doge_compiler/
token.rs

1use num_bigint::BigInt;
2
3use crate::ast::BinOp;
4
5/// A 1-based source position pointing at the first character of a token.
6#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
7pub struct Span {
8    pub line: u32,
9    pub col: u32,
10}
11
12/// A lexed token: its kind plus where it started.
13#[derive(Debug, Clone, PartialEq)]
14pub struct Token {
15    pub kind: TokenKind,
16    pub span: Span,
17}
18
19/// One piece of an interpolated string literal (`"a {b} c"`): either literal
20/// text or a `{…}` hole already lexed into its own token stream (with real
21/// source spans, so downstream diagnostics point at the right column).
22#[derive(Debug, Clone, PartialEq)]
23pub enum StrSegment {
24    Lit(String),
25    Hole(Vec<Token>),
26}
27
28/// Every kind of token Doge source can produce.
29#[derive(Debug, Clone, PartialEq)]
30pub enum TokenKind {
31    Pls,
32    Bork,
33    Bonk,
34    Bark,
35    Wow,
36    Such,
37    Much,
38    Many,
39    So,
40    Very,
41    /// `super` — call a method inherited from the enclosing class's parent.
42    Super,
43    /// The fused `oh no` compound keyword
44    OhNo,
45
46    // Universal keywords
47    If,
48    Elif,
49    Else,
50    For,
51    While,
52    In,
53    Return,
54    Continue,
55    And,
56    Or,
57    Not,
58    True,
59    False,
60    None,
61
62    // Reserved words
63    Def,
64    Class,
65    Amaze,
66
67    // --- Literals and identifiers ---
68    Ident(String),
69    /// An integer literal at full width: `Int` is arbitrary precision, so a literal
70    /// larger than `i64` must survive to codegen intact.
71    Int(BigInt),
72    Float(f64),
73    Str(String),
74    /// A string literal containing at least one `{…}` interpolation hole.
75    StrInterp(Vec<StrSegment>),
76
77    // --- Operators ---
78    Plus,
79    Minus,
80    Star,
81    StarStar,
82    Slash,
83    SlashSlash,
84    Percent,
85    Amp,
86    Pipe,
87    Caret,
88    Tilde,
89    Shl,
90    Shr,
91    EqEq,
92    NotEq,
93    Lt,
94    LtEq,
95    Gt,
96    GtEq,
97    Eq,
98    /// A compound assignment `op=`, e.g. `+=`, `//=`, `<<=`. Carries the binary
99    /// operator applied before the store.
100    AugAssign(BinOp),
101    Colon,
102    Bang,
103    Comma,
104    Dot,
105    LParen,
106    RParen,
107    LBracket,
108    RBracket,
109    LBrace,
110    RBrace,
111
112    // --- Structural (synthesized by the lexer) ---
113    Newline,
114    Indent,
115    Dedent,
116    Eof,
117}
118
119impl TokenKind {
120    /// A short human-readable name for this kind, used when a diagnostic needs
121    /// to name the token it did not expect.
122    pub fn describe(&self) -> String {
123        match self {
124            // Keyword tokens carry their spelling in the KEYWORDS table, so the
125            // spelling lives in one place; a new keyword variant makes this match
126            // non-exhaustive until it is added here too.
127            TokenKind::Pls
128            | TokenKind::Bork
129            | TokenKind::Bonk
130            | TokenKind::Bark
131            | TokenKind::Wow
132            | TokenKind::Such
133            | TokenKind::Much
134            | TokenKind::Many
135            | TokenKind::So
136            | TokenKind::Very
137            | TokenKind::Super
138            | TokenKind::If
139            | TokenKind::Elif
140            | TokenKind::Else
141            | TokenKind::For
142            | TokenKind::While
143            | TokenKind::In
144            | TokenKind::Return
145            | TokenKind::Continue
146            | TokenKind::And
147            | TokenKind::Or
148            | TokenKind::Not
149            | TokenKind::True
150            | TokenKind::False
151            | TokenKind::None
152            | TokenKind::Def
153            | TokenKind::Class
154            | TokenKind::Amaze => crate::keywords::keyword_spelling(self)
155                .expect("compiler bug: keyword token missing from KEYWORDS table")
156                .into(),
157            TokenKind::OhNo => "oh no".into(),
158            TokenKind::Ident(name) => format!("name '{name}'"),
159            TokenKind::Int(n) => format!("the number {n}"),
160            TokenKind::Float(f) => format!("the number {f}"),
161            TokenKind::Str(_) | TokenKind::StrInterp(_) => "a string".into(),
162            TokenKind::Plus => "+".into(),
163            TokenKind::Minus => "-".into(),
164            TokenKind::Star => "*".into(),
165            TokenKind::StarStar => "**".into(),
166            TokenKind::Slash => "/".into(),
167            TokenKind::SlashSlash => "//".into(),
168            TokenKind::Percent => "%".into(),
169            TokenKind::Amp => "&".into(),
170            TokenKind::Pipe => "|".into(),
171            TokenKind::Caret => "^".into(),
172            TokenKind::Tilde => "~".into(),
173            TokenKind::Shl => "<<".into(),
174            TokenKind::Shr => ">>".into(),
175            TokenKind::EqEq => "==".into(),
176            TokenKind::NotEq => "!=".into(),
177            TokenKind::Lt => "<".into(),
178            TokenKind::LtEq => "<=".into(),
179            TokenKind::Gt => ">".into(),
180            TokenKind::GtEq => ">=".into(),
181            TokenKind::Eq => "=".into(),
182            TokenKind::AugAssign(op) => format!("{}=", op.symbol()),
183            TokenKind::Colon => ":".into(),
184            TokenKind::Bang => "!".into(),
185            TokenKind::Comma => ",".into(),
186            TokenKind::Dot => ".".into(),
187            TokenKind::LParen => "(".into(),
188            TokenKind::RParen => ")".into(),
189            TokenKind::LBracket => "[".into(),
190            TokenKind::RBracket => "]".into(),
191            TokenKind::LBrace => "{".into(),
192            TokenKind::RBrace => "}".into(),
193            TokenKind::Newline => "the end of the line".into(),
194            TokenKind::Indent => "more indentation".into(),
195            TokenKind::Dedent => "less indentation".into(),
196            TokenKind::Eof => "the end of the script".into(),
197        }
198    }
199}
200
201#[cfg(test)]
202mod tests {
203    use super::*;
204    use crate::keywords::{keyword_spelling, lookup, KEYWORDS};
205
206    #[test]
207    fn every_keyword_round_trips_through_lookup_and_describe() {
208        for (spelling, kind) in KEYWORDS {
209            assert_eq!(lookup(spelling).as_ref(), Some(kind));
210            assert_eq!(keyword_spelling(kind), Some(*spelling));
211            assert_eq!(kind.describe(), *spelling);
212        }
213    }
214
215    #[test]
216    fn non_keyword_tokens_have_no_keyword_spelling() {
217        assert_eq!(keyword_spelling(&TokenKind::OhNo), None);
218        assert_eq!(keyword_spelling(&TokenKind::Plus), None);
219        assert_eq!(keyword_spelling(&TokenKind::Eof), None);
220    }
221
222    #[test]
223    fn ohno_describes_as_two_words() {
224        assert_eq!(TokenKind::OhNo.describe(), "oh no");
225    }
226}
doge_compiler/token.rs

doge_compiler/
token.rs
