solar_ast/
token.rs

1//! Solidity source code token.
2
3use crate::{
4    DocComment, StrKind,
5    ast::{BinOp, BinOpKind, UnOp, UnOpKind},
6};
7use solar_interface::{Ident, Span, Symbol, diagnostics::ErrorGuaranteed};
8use std::{borrow::Cow, fmt};
9
10/// The type of a comment.
11#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
12pub enum CommentKind {
13    /// `// ...`, `/// ...`
14    Line,
15    /// `/* ... */`, `/** ... */`
16    Block,
17}
18
19/// A binary operation token.
20///
21/// Note that this enum contains only binary operators that can also be used in assignments.
22#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
23pub enum BinOpToken {
24    /// `+`
25    Plus,
26    /// `-`
27    Minus,
28    /// `*`
29    Star,
30    /// `/`
31    Slash,
32    /// `%`
33    Percent,
34    /// `^`
35    Caret,
36    /// `&`
37    And,
38    /// `|`
39    Or,
40    /// `<<`
41    Shl,
42    /// `>>`
43    Shr,
44    /// `>>>`
45    Sar,
46}
47
48impl fmt::Display for BinOpToken {
49    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
50        f.write_str(self.to_str())
51    }
52}
53
54impl BinOpToken {
55    /// Returns the string representation of the binary operator token.
56    pub const fn to_str(self) -> &'static str {
57        match self {
58            Self::Plus => "+",
59            Self::Minus => "-",
60            Self::Star => "*",
61            Self::Slash => "/",
62            Self::Percent => "%",
63            Self::Caret => "^",
64            Self::And => "&",
65            Self::Or => "|",
66            Self::Shl => "<<",
67            Self::Shr => ">>",
68            Self::Sar => ">>>",
69        }
70    }
71
72    /// Returns the string representation of the binary operator token followed by an equals sign
73    /// (`=`).
74    pub const fn to_str_with_eq(self) -> &'static str {
75        match self {
76            Self::Plus => "+=",
77            Self::Minus => "-=",
78            Self::Star => "*=",
79            Self::Slash => "/=",
80            Self::Percent => "%=",
81            Self::Caret => "^=",
82            Self::And => "&=",
83            Self::Or => "|=",
84            Self::Shl => "<<=",
85            Self::Shr => ">>=",
86            Self::Sar => ">>>=",
87        }
88    }
89
90    /// Returns the binary operator kind.
91    #[inline]
92    pub const fn as_binop(self) -> BinOpKind {
93        match self {
94            Self::Plus => BinOpKind::Add,
95            Self::Minus => BinOpKind::Sub,
96            Self::Star => BinOpKind::Mul,
97            Self::Slash => BinOpKind::Div,
98            Self::Percent => BinOpKind::Rem,
99            Self::Caret => BinOpKind::BitXor,
100            Self::And => BinOpKind::BitAnd,
101            Self::Or => BinOpKind::BitOr,
102            Self::Shl => BinOpKind::Shl,
103            Self::Shr => BinOpKind::Shr,
104            Self::Sar => BinOpKind::Sar,
105        }
106    }
107}
108
109/// Describes how a sequence of token trees is delimited.
110#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
111pub enum Delimiter {
112    /// `( ... )`
113    Parenthesis,
114    /// `{ ... }`
115    Brace,
116    /// `[ ... ]`
117    Bracket,
118}
119
120impl Delimiter {
121    /// Returns the string representation of the opening delimiter.
122    pub const fn to_open_str(self) -> &'static str {
123        match self {
124            Self::Parenthesis => "(",
125            Self::Brace => "{",
126            Self::Bracket => "[",
127        }
128    }
129
130    /// Returns the string representation of the closing delimiter.
131    pub const fn to_close_str(self) -> &'static str {
132        match self {
133            Self::Parenthesis => ")",
134            Self::Brace => "}",
135            Self::Bracket => "]",
136        }
137    }
138}
139
140/// A literal token. Different from an AST literal as this is unparsed and only contains the raw
141/// contents.
142#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
143pub struct TokenLit {
144    /// The symbol of the literal token, excluding any quotes.
145    pub symbol: Symbol,
146    /// The literal kind.
147    pub kind: TokenLitKind,
148}
149
150impl fmt::Display for TokenLit {
151    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
152        let &Self { kind, symbol } = self;
153        match kind {
154            TokenLitKind::Str => write!(f, "\"{symbol}\""),
155            TokenLitKind::UnicodeStr => write!(f, "unicode\"{symbol}\""),
156            TokenLitKind::HexStr => write!(f, "hex\"{symbol}\""),
157            TokenLitKind::Integer | TokenLitKind::Rational | TokenLitKind::Err(_) => {
158                write!(f, "{symbol}")
159            }
160        }
161    }
162}
163
164impl TokenLit {
165    /// Creates a new literal token.
166    #[inline]
167    pub const fn new(kind: TokenLitKind, symbol: Symbol) -> Self {
168        Self { kind, symbol }
169    }
170
171    /// Returns a description of the literal.
172    pub const fn description(self) -> &'static str {
173        self.kind.description()
174    }
175}
176
177/// A kind of literal token.
178#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
179pub enum TokenLitKind {
180    /// An integer literal token.
181    Integer,
182    /// A rational literal token.
183    Rational,
184    /// A string literal token.
185    Str,
186    /// A unicode string literal token.
187    UnicodeStr,
188    /// A hex string literal token.
189    HexStr,
190    /// An error occurred while lexing the literal token.
191    Err(ErrorGuaranteed),
192}
193
194impl From<StrKind> for TokenLitKind {
195    fn from(str_kind: StrKind) -> Self {
196        match str_kind {
197            StrKind::Str => Self::Str,
198            StrKind::Unicode => Self::UnicodeStr,
199            StrKind::Hex => Self::HexStr,
200        }
201    }
202}
203
204impl TokenLitKind {
205    /// Returns the description of the literal kind.
206    pub const fn description(self) -> &'static str {
207        match self {
208            Self::Integer => "integer",
209            Self::Rational => "rational",
210            Self::Str => "string",
211            Self::UnicodeStr => "unicode string",
212            Self::HexStr => "hex string",
213            Self::Err(_) => "error",
214        }
215    }
216}
217
218/// A kind of token.
219#[derive(Clone, Copy, Debug, PartialEq, Eq)]
220pub enum TokenKind {
221    // Expression-operator symbols.
222    /// `=`
223    Eq,
224    /// `<`
225    Lt,
226    /// `<=`
227    Le,
228    /// `==`
229    EqEq,
230    /// `!=`
231    Ne,
232    /// `>=`
233    Ge,
234    /// `>`
235    Gt,
236    /// `&&`
237    AndAnd,
238    /// `||`
239    OrOr,
240    /// `!`
241    Not,
242    /// `~`
243    Tilde,
244    /// `:=`
245    Walrus,
246    /// `++`
247    PlusPlus,
248    /// `--`
249    MinusMinus,
250    /// `**`
251    StarStar,
252    /// A binary operator token.
253    BinOp(BinOpToken),
254    /// A binary operator token, followed by an equals sign (`=`).
255    BinOpEq(BinOpToken),
256
257    // Structural symbols.
258    /// `@`
259    At,
260    /// `.`
261    Dot,
262    /// `,`
263    Comma,
264    /// `;`
265    Semi,
266    /// `:`
267    Colon,
268    /// `->`
269    Arrow,
270    /// `=>`
271    FatArrow,
272    /// `?`
273    Question,
274    /// An opening delimiter (e.g., `{`).
275    OpenDelim(Delimiter),
276    /// A closing delimiter (e.g., `}`).
277    CloseDelim(Delimiter),
278
279    // Literals.
280    /// A literal token.
281    ///
282    /// Note that this does not include boolean literals.
283    ///
284    /// `Symbol` is the literal's parsed data. In string literals, this is the unescaped value, and
285    /// excludes its quotes (`"`, `'`) and prefix (`hex`, `unicode`).
286    Literal(TokenLitKind, Symbol),
287
288    /// Identifier token.
289    Ident(Symbol),
290
291    /// A comment or doc-comment token.
292    ///
293    /// `Symbol` is the comment's data excluding its "quotes" (`//`, `/**`)
294    /// similarly to symbols in string literal tokens.
295    Comment(bool /* is_doc */, CommentKind, Symbol),
296
297    /// End of file marker.
298    Eof,
299}
300
301impl fmt::Display for TokenKind {
302    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
303        f.write_str(&self.description())
304    }
305}
306
307impl TokenKind {
308    /// Creates a new literal token kind.
309    pub fn lit(kind: TokenLitKind, symbol: Symbol) -> Self {
310        Self::Literal(kind, symbol)
311    }
312
313    /// Returns the string representation of the token kind.
314    pub fn as_str(&self) -> &str {
315        match self {
316            Self::Eq => "=",
317            Self::Lt => "<",
318            Self::Le => "<=",
319            Self::EqEq => "==",
320            Self::Ne => "!=",
321            Self::Ge => ">=",
322            Self::Gt => ">",
323            Self::AndAnd => "&&",
324            Self::OrOr => "||",
325            Self::Not => "!",
326            Self::Tilde => "~",
327            Self::Walrus => ":=",
328            Self::PlusPlus => "++",
329            Self::MinusMinus => "--",
330            Self::StarStar => "**",
331            Self::BinOp(op) => op.to_str(),
332            Self::BinOpEq(op) => op.to_str_with_eq(),
333
334            Self::At => "@",
335            Self::Dot => ".",
336            Self::Comma => ",",
337            Self::Semi => ";",
338            Self::Colon => ":",
339            Self::Arrow => "->",
340            Self::FatArrow => "=>",
341            Self::Question => "?",
342            Self::OpenDelim(d) => d.to_open_str(),
343            Self::CloseDelim(d) => d.to_close_str(),
344
345            Self::Literal(.., symbol) | Self::Ident(.., symbol) | Self::Comment(.., symbol) => {
346                symbol.as_str()
347            }
348
349            Self::Eof => "<eof>",
350        }
351    }
352
353    /// Returns the description of the token kind.
354    pub fn description(&self) -> Cow<'_, str> {
355        match self {
356            Self::Literal(kind, _) => return format!("<{}>", kind.description()).into(),
357            Self::Ident(symbol) => return symbol.to_string().into(),
358            Self::Comment(false, CommentKind::Block, _) => "<block comment>",
359            Self::Comment(true, CommentKind::Block, _) => "<block doc-comment>",
360            Self::Comment(false, CommentKind::Line, _) => "<line comment>",
361            Self::Comment(true, CommentKind::Line, _) => "<line doc-comment>",
362            _ => self.as_str(),
363        }
364        .into()
365    }
366
367    /// Returns `true` if the token kind is an operator.
368    pub const fn is_op(&self) -> bool {
369        use TokenKind::*;
370        match self {
371            Eq | Lt | Le | EqEq | Ne | Ge | Gt | AndAnd | OrOr | Not | Tilde | Walrus
372            | PlusPlus | MinusMinus | StarStar | BinOp(_) | BinOpEq(_) | At | Dot | Comma
373            | Colon | Arrow | FatArrow | Question => true,
374
375            OpenDelim(..) | CloseDelim(..) | Literal(..) | Comment(..) | Ident(..) | Semi | Eof => {
376                false
377            }
378        }
379    }
380
381    /// Returns the token kind as a unary operator, if any.
382    pub fn as_unop(&self, is_postfix: bool) -> Option<UnOpKind> {
383        let kind = if is_postfix {
384            match self {
385                Self::PlusPlus => UnOpKind::PostInc,
386                Self::MinusMinus => UnOpKind::PostDec,
387                _ => return None,
388            }
389        } else {
390            match self {
391                Self::PlusPlus => UnOpKind::PreInc,
392                Self::MinusMinus => UnOpKind::PreDec,
393                Self::Not => UnOpKind::Not,
394                Self::Tilde => UnOpKind::BitNot,
395                Self::BinOp(BinOpToken::Minus) => UnOpKind::Neg,
396                _ => return None,
397            }
398        };
399        debug_assert_eq!(kind.is_postfix(), is_postfix);
400        Some(kind)
401    }
402
403    /// Returns the token kind as a binary operator, if any.
404    #[inline]
405    pub fn as_binop(&self) -> Option<BinOpKind> {
406        match self {
407            Self::Eq => Some(BinOpKind::Eq),
408            Self::Lt => Some(BinOpKind::Lt),
409            Self::Le => Some(BinOpKind::Le),
410            Self::EqEq => Some(BinOpKind::Eq),
411            Self::Ne => Some(BinOpKind::Ne),
412            Self::Ge => Some(BinOpKind::Ge),
413            Self::Gt => Some(BinOpKind::Gt),
414            Self::AndAnd => Some(BinOpKind::And),
415            Self::OrOr => Some(BinOpKind::Or),
416            Self::StarStar => Some(BinOpKind::Pow),
417            Self::BinOp(op) => Some(op.as_binop()),
418            _ => None,
419        }
420    }
421
422    /// Returns the token kind as a binary operator, if any.
423    #[inline]
424    pub fn as_binop_eq(&self) -> Option<BinOpKind> {
425        match self {
426            Self::BinOpEq(op) => Some(op.as_binop()),
427            _ => None,
428        }
429    }
430
431    /// Returns `true` if the token kind is a normal comment (not a doc-comment).
432    #[inline]
433    pub const fn is_comment(&self) -> bool {
434        matches!(self, Self::Comment(false, ..))
435    }
436
437    /// Returns `true` if the token kind is a comment or doc-comment.
438    #[inline]
439    pub const fn is_comment_or_doc(&self) -> bool {
440        matches!(self, Self::Comment(..))
441    }
442
443    /// Glues two token kinds together.
444    pub const fn glue(self, other: Self) -> Option<Self> {
445        use BinOpToken::*;
446        use TokenKind::*;
447        Some(match self {
448            Eq => match other {
449                Eq => EqEq,
450                Gt => FatArrow,
451                _ => return None,
452            },
453            Lt => match other {
454                Eq => Le,
455                Lt => BinOp(Shl),
456                Le => BinOpEq(Shl),
457                _ => return None,
458            },
459            Gt => match other {
460                Eq => Ge,
461                Gt => BinOp(Shr),
462                Ge => BinOpEq(Shr),
463                BinOp(Shr) => BinOp(Sar),
464                BinOpEq(Shr) => BinOpEq(Sar),
465                _ => return None,
466            },
467            Not => match other {
468                Eq => Ne,
469                _ => return None,
470            },
471            Colon => match other {
472                Eq => Walrus,
473                _ => return None,
474            },
475            BinOp(op) => match (op, other) {
476                (op, Eq) => BinOpEq(op),
477                (And, BinOp(And)) => AndAnd,
478                (Or, BinOp(Or)) => OrOr,
479                (Minus, Gt) => Arrow,
480                (Shr, Gt) => BinOp(Sar),
481                (Shr, Ge) => BinOpEq(Sar),
482                (Plus, BinOp(Plus)) => PlusPlus,
483                (Minus, BinOp(Minus)) => MinusMinus,
484                (Star, BinOp(Star)) => StarStar,
485                _ => return None,
486            },
487
488            Le | EqEq | Ne | Ge | AndAnd | OrOr | Tilde | Walrus | PlusPlus | MinusMinus
489            | StarStar | BinOpEq(_) | At | Dot | Comma | Semi | Arrow | FatArrow | Question
490            | OpenDelim(_) | CloseDelim(_) | Literal(..) | Ident(_) | Comment(..) | Eof => {
491                return None;
492            }
493        })
494    }
495}
496
497/// A single token.
498#[derive(Clone, Copy, Debug, PartialEq, Eq)]
499pub struct Token {
500    /// The kind of the token.
501    pub kind: TokenKind,
502    /// The full span of the token.
503    pub span: Span,
504}
505
506impl From<Ident> for Token {
507    #[inline]
508    fn from(ident: Ident) -> Self {
509        Self::from_ast_ident(ident)
510    }
511}
512
513impl Token {
514    /// The [EOF](TokenKind::Eof) token.
515    pub const EOF: Self = Self::new(TokenKind::Eof, Span::DUMMY);
516
517    /// A dummy token that will be thrown away later.
518    pub const DUMMY: Self = Self::new(TokenKind::Question, Span::DUMMY);
519
520    /// Creates a new token.
521    #[inline]
522    pub const fn new(kind: TokenKind, span: Span) -> Self {
523        Self { kind, span }
524    }
525
526    /// Recovers a `Token` from an `Ident`.
527    #[inline]
528    pub fn from_ast_ident(ident: Ident) -> Self {
529        Self::new(TokenKind::Ident(ident.name), ident.span)
530    }
531
532    /// Creates a new identifier if the kind is [`TokenKind::Ident`].
533    #[inline]
534    pub const fn ident(&self) -> Option<Ident> {
535        match self.kind {
536            TokenKind::Ident(ident) => Some(Ident::new(ident, self.span)),
537            _ => None,
538        }
539    }
540
541    /// Returns the literal if the kind is [`TokenKind::Literal`].
542    #[inline]
543    pub const fn lit(&self) -> Option<TokenLit> {
544        match self.kind {
545            TokenKind::Literal(kind, symbol) => Some(TokenLit::new(kind, symbol)),
546            _ => None,
547        }
548    }
549
550    /// Returns this token's literal kind, if any.
551    #[inline]
552    pub const fn lit_kind(&self) -> Option<TokenLitKind> {
553        match self.kind {
554            TokenKind::Literal(kind, _) => Some(kind),
555            _ => None,
556        }
557    }
558
559    /// Returns the comment if the kind is [`TokenKind::Comment`], and whether it's a doc-comment.
560    #[inline]
561    pub const fn comment(&self) -> Option<(bool, DocComment)> {
562        match self.kind {
563            TokenKind::Comment(is_doc, kind, symbol) => {
564                Some((is_doc, DocComment { span: self.span, kind, symbol }))
565            }
566            _ => None,
567        }
568    }
569
570    /// Returns the comment if the kind is [`TokenKind::Comment`].
571    ///
572    /// Does not check that `is_doc` is `true`.
573    #[inline]
574    pub const fn doc(&self) -> Option<DocComment> {
575        match self.kind {
576            TokenKind::Comment(_, kind, symbol) => {
577                Some(DocComment { span: self.span, kind, symbol })
578            }
579            _ => None,
580        }
581    }
582
583    /// Returns `true` if the token is an operator.
584    #[inline]
585    pub const fn is_op(&self) -> bool {
586        self.kind.is_op()
587    }
588
589    /// Returns the token as a unary operator, if any.
590    #[inline]
591    pub fn as_unop(&self, is_postfix: bool) -> Option<UnOp> {
592        self.kind.as_unop(is_postfix).map(|kind| UnOp { span: self.span, kind })
593    }
594
595    /// Returns the token as a binary operator, if any.
596    #[inline]
597    pub fn as_binop(&self) -> Option<BinOp> {
598        self.kind.as_binop().map(|kind| BinOp { span: self.span, kind })
599    }
600
601    /// Returns the token as a binary operator, if any.
602    #[inline]
603    pub fn as_binop_eq(&self) -> Option<BinOp> {
604        self.kind.as_binop_eq().map(|kind| BinOp { span: self.span, kind })
605    }
606
607    /// Returns `true` if the token is an identifier.
608    #[inline]
609    pub const fn is_ident(&self) -> bool {
610        matches!(self.kind, TokenKind::Ident(_))
611    }
612
613    /// Returns `true` if the token is a literal. Includes `bool` literals.
614    #[inline]
615    pub fn is_lit(&self) -> bool {
616        matches!(self.kind, TokenKind::Literal(..)) || self.is_bool_lit()
617    }
618
619    /// Returns `true` if the token is a given keyword, `kw`.
620    #[inline]
621    pub fn is_keyword(&self, kw: Symbol) -> bool {
622        self.is_ident_where(|id| id.name == kw)
623    }
624
625    /// Returns `true` if the token is any of the given keywords.
626    #[inline]
627    pub fn is_keyword_any(&self, kws: &[Symbol]) -> bool {
628        self.is_ident_where(|id| kws.contains(&id.name))
629    }
630
631    /// Returns `true` if the token is a keyword used in the language.
632    #[inline]
633    pub fn is_used_keyword(&self) -> bool {
634        self.is_ident_where(Ident::is_used_keyword)
635    }
636
637    /// Returns `true` if the token is a keyword reserved for possible future use.
638    #[inline]
639    pub fn is_unused_keyword(&self) -> bool {
640        self.is_ident_where(Ident::is_unused_keyword)
641    }
642
643    /// Returns `true` if the token is a keyword.
644    #[inline]
645    pub fn is_reserved_ident(&self, yul: bool) -> bool {
646        self.is_ident_where(|i| i.is_reserved(yul))
647    }
648
649    /// Returns `true` if the token is an identifier, but not a keyword.
650    #[inline]
651    pub fn is_non_reserved_ident(&self, yul: bool) -> bool {
652        self.is_ident_where(|i| i.is_non_reserved(yul))
653    }
654
655    /// Returns `true` if the token is an elementary type name.
656    ///
657    /// Note that this does not include `[u]fixedMxN` types.
658    #[inline]
659    pub fn is_elementary_type(&self) -> bool {
660        self.is_ident_where(Ident::is_elementary_type)
661    }
662
663    /// Returns `true` if the token is the identifier `true` or `false`.
664    #[inline]
665    pub fn is_bool_lit(&self) -> bool {
666        self.is_ident_where(|id| id.name.is_bool_lit())
667    }
668
669    /// Returns `true` if the token is a numeric literal.
670    #[inline]
671    pub fn is_numeric_lit(&self) -> bool {
672        matches!(self.kind, TokenKind::Literal(TokenLitKind::Integer | TokenLitKind::Rational, _))
673    }
674
675    /// Returns `true` if the token is the integer literal.
676    #[inline]
677    pub fn is_integer_lit(&self) -> bool {
678        matches!(self.kind, TokenKind::Literal(TokenLitKind::Integer, _))
679    }
680
681    /// Returns `true` if the token is the rational literal.
682    #[inline]
683    pub fn is_rational_lit(&self) -> bool {
684        matches!(self.kind, TokenKind::Literal(TokenLitKind::Rational, _))
685    }
686
687    /// Returns `true` if the token is a string literal.
688    #[inline]
689    pub fn is_str_lit(&self) -> bool {
690        matches!(self.kind, TokenKind::Literal(TokenLitKind::Str, _))
691    }
692
693    /// Returns `true` if the token is an identifier for which `pred` holds.
694    #[inline]
695    pub fn is_ident_where(&self, pred: impl FnOnce(Ident) -> bool) -> bool {
696        self.ident().map(pred).unwrap_or(false)
697    }
698
699    /// Returns `true` if the token is an end-of-file marker.
700    #[inline]
701    pub const fn is_eof(&self) -> bool {
702        matches!(self.kind, TokenKind::Eof)
703    }
704
705    /// Returns `true` if the token is the given open delimiter.
706    #[inline]
707    pub fn is_open_delim(&self, d: Delimiter) -> bool {
708        self.kind == TokenKind::OpenDelim(d)
709    }
710
711    /// Returns `true` if the token is the given close delimiter.
712    #[inline]
713    pub fn is_close_delim(&self, d: Delimiter) -> bool {
714        self.kind == TokenKind::CloseDelim(d)
715    }
716
717    /// Returns `true` if the token is a normal comment (not a doc-comment).
718    #[inline]
719    pub const fn is_comment(&self) -> bool {
720        self.kind.is_comment()
721    }
722
723    /// Returns `true` if the token is a comment or doc-comment.
724    #[inline]
725    pub const fn is_comment_or_doc(&self) -> bool {
726        self.kind.is_comment_or_doc()
727    }
728
729    /// Returns `true` if the token is a location specifier.
730    #[inline]
731    pub fn is_location_specifier(&self) -> bool {
732        self.is_ident_where(Ident::is_location_specifier)
733    }
734
735    /// Returns `true` if the token is a mutability specifier.
736    #[inline]
737    pub fn is_mutability_specifier(&self) -> bool {
738        self.is_ident_where(Ident::is_mutability_specifier)
739    }
740
741    /// Returns `true` if the token is a visibility specifier.
742    #[inline]
743    pub fn is_visibility_specifier(&self) -> bool {
744        self.is_ident_where(Ident::is_visibility_specifier)
745    }
746
747    /// Returns this token's full description: `{self.description()} '{self.kind}'`.
748    pub fn full_description(&self) -> impl fmt::Display + '_ {
749        // https://github.com/rust-lang/rust/blob/44bf2a32a52467c45582c3355a893400e620d010/compiler/rustc_parse/src/parser/mod.rs#L378
750        if let Some(description) = self.description() {
751            format!("{description} `{}`", self.kind)
752        } else {
753            format!("`{}`", self.kind)
754        }
755    }
756
757    /// Returns the string representation of the token.
758    pub fn as_str(&self) -> &str {
759        self.kind.as_str()
760    }
761
762    /// Returns this token's description, if any.
763    #[inline]
764    pub fn description(self) -> Option<TokenDescription> {
765        TokenDescription::from_token(self)
766    }
767
768    /// Glues two tokens together.
769    pub fn glue(self, other: Self) -> Option<Self> {
770        self.kind.glue(other.kind).map(|kind| Self::new(kind, self.span.to(other.span)))
771    }
772}
773
774/// A description of a token.
775///
776/// Precedes the token string in error messages like `keyword 'for'` in `expected identifier, found
777/// keyword 'for'`. See [`full_description`](Token::full_description).
778#[derive(Clone, Copy, Debug, PartialEq, Eq)]
779pub enum TokenDescription {
780    /// A keyword.
781    Keyword,
782    /// A reserved keyword.
783    ReservedKeyword,
784    /// A Yul keyword.
785    YulKeyword,
786    /// A Yul EVM builtin.
787    YulEvmBuiltin,
788}
789
790impl fmt::Display for TokenDescription {
791    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
792        f.write_str(self.to_str())
793    }
794}
795
796impl TokenDescription {
797    /// Returns the description of the given token.
798    pub fn from_token(token: Token) -> Option<Self> {
799        match token.kind {
800            _ if token.is_used_keyword() => Some(Self::Keyword),
801            _ if token.is_unused_keyword() => Some(Self::ReservedKeyword),
802            _ if token.is_ident_where(|id| id.is_yul_keyword()) => Some(Self::YulKeyword),
803            _ if token.is_ident_where(|id| id.is_yul_evm_builtin()) => Some(Self::YulEvmBuiltin),
804            _ => None,
805        }
806    }
807
808    /// Returns the string representation of the token description.
809    pub const fn to_str(self) -> &'static str {
810        match self {
811            Self::Keyword => "keyword",
812            Self::ReservedKeyword => "reserved keyword",
813            Self::YulKeyword => "Yul keyword",
814            Self::YulEvmBuiltin => "Yul EVM builtin keyword",
815        }
816    }
817}
solar_ast/token.rs

solar_ast/
token.rs
