doge_compiler/
token.rs

1use num_bigint::BigInt;
2
3use crate::ast::BinOp;
4
5/// A 1-based source position pointing at the first character of a token.
6#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
7pub struct Span {
8    pub line: u32,
9    pub col: u32,
10}
11
12/// A lexed token: its kind plus where it started.
13#[derive(Debug, Clone, PartialEq)]
14pub struct Token {
15    pub kind: TokenKind,
16    pub span: Span,
17}
18
19/// One piece of an interpolated string literal (`"a {b} c"`): either literal
20/// text or a `{…}` hole already lexed into its own token stream (with real
21/// source spans, so downstream diagnostics point at the right column).
22#[derive(Debug, Clone, PartialEq)]
23pub enum StrSegment {
24    Lit(String),
25    Hole(Vec<Token>),
26}
27
28/// Every kind of token Doge source can produce.
29#[derive(Debug, Clone, PartialEq)]
30pub enum TokenKind {
31    Pls,
32    Bork,
33    Bonk,
34    Bark,
35    Wow,
36    Such,
37    Much,
38    Many,
39    So,
40    Very,
41    /// `super` — call a method inherited from the enclosing class's parent.
42    Super,
43    /// The fused `oh no` compound keyword
44    OhNo,
45
46    If,
47    Elif,
48    Else,
49    For,
50    While,
51    In,
52    Return,
53    Continue,
54    And,
55    Or,
56    Not,
57    True,
58    False,
59    None,
60
61    Def,
62    Class,
63    Amaze,
64
65    Ident(String),
66    /// An integer literal at full width: `Int` is arbitrary precision, so a literal
67    /// larger than `i64` must survive to codegen intact.
68    Int(BigInt),
69    Float(f64),
70    Str(String),
71    /// A string literal containing at least one `{…}` interpolation hole.
72    StrInterp(Vec<StrSegment>),
73
74    Plus,
75    Minus,
76    Star,
77    StarStar,
78    Slash,
79    SlashSlash,
80    Percent,
81    Amp,
82    Pipe,
83    Caret,
84    Tilde,
85    Shl,
86    Shr,
87    EqEq,
88    NotEq,
89    Lt,
90    LtEq,
91    Gt,
92    GtEq,
93    Eq,
94    /// A compound assignment `op=`, e.g. `+=`, `//=`, `<<=`. Carries the binary
95    /// operator applied before the store.
96    AugAssign(BinOp),
97    Colon,
98    Bang,
99    Comma,
100    Dot,
101    LParen,
102    RParen,
103    LBracket,
104    RBracket,
105    LBrace,
106    RBrace,
107
108    Newline,
109    Indent,
110    Dedent,
111    Eof,
112}
113
114impl TokenKind {
115    /// A short human-readable name for this kind, used when a diagnostic needs
116    /// to name the token it did not expect.
117    pub fn describe(&self) -> String {
118        match self {
119            // Keyword tokens carry their spelling in the KEYWORDS table, so the
120            // spelling lives in one place; a new keyword variant makes this match
121            // non-exhaustive until it is added here too.
122            TokenKind::Pls
123            | TokenKind::Bork
124            | TokenKind::Bonk
125            | TokenKind::Bark
126            | TokenKind::Wow
127            | TokenKind::Such
128            | TokenKind::Much
129            | TokenKind::Many
130            | TokenKind::So
131            | TokenKind::Very
132            | TokenKind::Super
133            | TokenKind::If
134            | TokenKind::Elif
135            | TokenKind::Else
136            | TokenKind::For
137            | TokenKind::While
138            | TokenKind::In
139            | TokenKind::Return
140            | TokenKind::Continue
141            | TokenKind::And
142            | TokenKind::Or
143            | TokenKind::Not
144            | TokenKind::True
145            | TokenKind::False
146            | TokenKind::None
147            | TokenKind::Def
148            | TokenKind::Class
149            | TokenKind::Amaze => crate::keywords::keyword_spelling(self)
150                .expect("compiler bug: keyword token missing from KEYWORDS table")
151                .into(),
152            TokenKind::OhNo => "oh no".into(),
153            TokenKind::Ident(name) => format!("name '{name}'"),
154            TokenKind::Int(n) => format!("the number {n}"),
155            TokenKind::Float(f) => format!("the number {f}"),
156            TokenKind::Str(_) | TokenKind::StrInterp(_) => "a string".into(),
157            TokenKind::Plus => "+".into(),
158            TokenKind::Minus => "-".into(),
159            TokenKind::Star => "*".into(),
160            TokenKind::StarStar => "**".into(),
161            TokenKind::Slash => "/".into(),
162            TokenKind::SlashSlash => "//".into(),
163            TokenKind::Percent => "%".into(),
164            TokenKind::Amp => "&".into(),
165            TokenKind::Pipe => "|".into(),
166            TokenKind::Caret => "^".into(),
167            TokenKind::Tilde => "~".into(),
168            TokenKind::Shl => "<<".into(),
169            TokenKind::Shr => ">>".into(),
170            TokenKind::EqEq => "==".into(),
171            TokenKind::NotEq => "!=".into(),
172            TokenKind::Lt => "<".into(),
173            TokenKind::LtEq => "<=".into(),
174            TokenKind::Gt => ">".into(),
175            TokenKind::GtEq => ">=".into(),
176            TokenKind::Eq => "=".into(),
177            TokenKind::AugAssign(op) => format!("{}=", op.symbol()),
178            TokenKind::Colon => ":".into(),
179            TokenKind::Bang => "!".into(),
180            TokenKind::Comma => ",".into(),
181            TokenKind::Dot => ".".into(),
182            TokenKind::LParen => "(".into(),
183            TokenKind::RParen => ")".into(),
184            TokenKind::LBracket => "[".into(),
185            TokenKind::RBracket => "]".into(),
186            TokenKind::LBrace => "{".into(),
187            TokenKind::RBrace => "}".into(),
188            TokenKind::Newline => "the end of the line".into(),
189            TokenKind::Indent => "more indentation".into(),
190            TokenKind::Dedent => "less indentation".into(),
191            TokenKind::Eof => "the end of the script".into(),
192        }
193    }
194}
195
196#[cfg(test)]
197mod tests {
198    use super::*;
199    use crate::keywords::{keyword_spelling, lookup, KEYWORDS};
200
201    #[test]
202    fn every_keyword_round_trips_through_lookup_and_describe() {
203        for (spelling, kind) in KEYWORDS {
204            assert_eq!(lookup(spelling).as_ref(), Some(kind));
205            assert_eq!(keyword_spelling(kind), Some(*spelling));
206            assert_eq!(kind.describe(), *spelling);
207        }
208    }
209
210    #[test]
211    fn non_keyword_tokens_have_no_keyword_spelling() {
212        assert_eq!(keyword_spelling(&TokenKind::OhNo), None);
213        assert_eq!(keyword_spelling(&TokenKind::Plus), None);
214        assert_eq!(keyword_spelling(&TokenKind::Eof), None);
215    }
216
217    #[test]
218    fn ohno_describes_as_two_words() {
219        assert_eq!(TokenKind::OhNo.describe(), "oh no");
220    }
221}
doge_compiler/token.rs

doge_compiler/
token.rs
