use crate::span::Span;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum TokenKind {
Ident,
Auto,
Break,
Case,
Char,
Const,
Continue,
Default,
Do,
Double,
Else,
Enum,
Extern,
Float,
For,
Goto,
If,
Inline,
Int,
Long,
Register,
Restrict,
Return,
Short,
Signed,
Sizeof,
Static,
Struct,
Switch,
Typedef,
Union,
Unsigned,
Void,
Volatile,
While,
Bool,
Complex,
Imaginary,
String,
True,
False,
Null,
Alignas,
Alignof,
And,
AndEq,
Asm,
AtomicCancel,
AtomicCommit,
AtomicNoexcept,
Bitand,
Bitor,
Catch,
Class,
Compl,
Concept,
ConstCast,
Constexpr,
Constinit,
Decltype,
CppDefault,
Delete,
DynamicCast,
Explicit,
Export,
Friend,
IfConstexpr,
CppInline,
Lambda,
Mutable,
Namespace,
New,
Noexcept,
Not,
NotEq,
Nullptr,
Operator,
Or,
OrEq,
Override,
Private,
Protected,
Public,
Reflexpr,
CppRegister,
ReinterpretCast,
Requires,
StaticCast,
StaticAssert,
Template,
This,
ThreadLocal,
Throw,
Try,
Typeid,
Typename,
Using,
Virtual,
CppVolatile,
CppWhile,
Xor,
XorEq,
CoAwait,
CoReturn,
CoYield,
Char16,
Char32,
WcharT,
Plus,
Minus,
Star,
Slash,
Percent,
PlusPlus,
MinusMinus,
Ampersand,
Pipe,
Caret,
Tilde,
Exclamation,
LtLt,
GtGt,
EqEq,
Ne,
Lt,
Gt,
LtEq,
GtEq,
AmpersandAmpersand,
PipePipe,
Arrow,
ArrowStar,
DotStar,
Colon,
ColonColon,
Semicolon,
Comma,
Question,
Dot,
DotDot,
DotDotDot,
LParen,
RParen,
LBrace,
RBrace,
LBracket,
RBracket,
Eq,
PlusEq,
MinusEq,
StarEq,
SlashEq,
PercentEq,
AmpersandEq,
PipeEq,
CaretEq,
LtLtEq,
GtGtEq,
Hash,
HashHash,
Newline,
At,
IntLit,
UIntLit,
LongLit,
ULongLit,
LongLongLit,
ULongLongLit,
FloatLit,
DoubleLit,
HexFloatLit,
CharLit,
Char16Lit,
Char32Lit,
WcharLit,
StringLit,
RawStringLit,
LStringLit,
U16StringLit,
U32StringLit,
WStringLit,
UserDefinedLit,
Eof,
Error,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Token {
pub kind: TokenKind,
pub span: Span,
pub value: String,
pub leading_comments: Vec<Comment>,
pub has_line_break: bool,
}
impl Token {
pub fn new(kind: TokenKind, span: Span) -> Self {
Token {
kind,
span,
value: String::new(),
leading_comments: Vec::new(),
has_line_break: false,
}
}
pub fn with_value(kind: TokenKind, span: Span, value: impl Into<String>) -> Self {
Token {
kind,
span,
value: value.into(),
leading_comments: Vec::new(),
has_line_break: false,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Comment {
pub kind: CommentKind,
pub span: Span,
pub text: String,
}
impl Comment {
pub fn new(kind: CommentKind, span: Span, text: impl Into<String>) -> Self {
Comment {
kind,
span,
text: text.into(),
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum CommentKind {
Line,
Block,
}