doge_compiler/
token.rs

1use crate::ast::BinOp;
2
3/// A 1-based source position pointing at the first character of a token.
4#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
5pub struct Span {
6    pub line: u32,
7    pub col: u32,
8}
9
10/// A lexed token: its kind plus where it started.
11#[derive(Debug, Clone, PartialEq)]
12pub struct Token {
13    pub kind: TokenKind,
14    pub span: Span,
15}
16
17/// One piece of an interpolated string literal (`"a {b} c"`): either literal
18/// text or a `{…}` hole already lexed into its own token stream (with real
19/// source spans, so downstream diagnostics point at the right column).
20#[derive(Debug, Clone, PartialEq)]
21pub enum StrSegment {
22    Lit(String),
23    Hole(Vec<Token>),
24}
25
26/// Every kind of token Doge source can produce.
27#[derive(Debug, Clone, PartialEq)]
28pub enum TokenKind {
29    Pls,
30    Bork,
31    Bonk,
32    Bark,
33    Wow,
34    Such,
35    Much,
36    Many,
37    So,
38    Very,
39    /// `super` — call a method inherited from the enclosing class's parent.
40    Super,
41    /// The fused `oh no` compound keyword
42    OhNo,
43
44    // Universal keywords
45    If,
46    Elif,
47    Else,
48    For,
49    While,
50    In,
51    Return,
52    Continue,
53    And,
54    Or,
55    Not,
56    True,
57    False,
58    None,
59
60    // Reserved words
61    Def,
62    Class,
63    Amaze,
64
65    // --- Literals and identifiers ---
66    Ident(String),
67    Int(i64),
68    Float(f64),
69    Str(String),
70    /// A string literal containing at least one `{…}` interpolation hole.
71    StrInterp(Vec<StrSegment>),
72
73    // --- Operators ---
74    Plus,
75    Minus,
76    Star,
77    StarStar,
78    Slash,
79    SlashSlash,
80    Percent,
81    Amp,
82    Pipe,
83    Caret,
84    Tilde,
85    Shl,
86    Shr,
87    EqEq,
88    NotEq,
89    Lt,
90    LtEq,
91    Gt,
92    GtEq,
93    Eq,
94    /// A compound assignment `op=`, e.g. `+=`, `//=`, `<<=`. Carries the binary
95    /// operator applied before the store.
96    AugAssign(BinOp),
97    Colon,
98    Bang,
99    Comma,
100    Dot,
101    LParen,
102    RParen,
103    LBracket,
104    RBracket,
105    LBrace,
106    RBrace,
107
108    // --- Structural (synthesized by the lexer) ---
109    Newline,
110    Indent,
111    Dedent,
112    Eof,
113}
114
115impl TokenKind {
116    /// A short human-readable name for this kind, used when a diagnostic needs
117    /// to name the token it did not expect.
118    pub fn describe(&self) -> String {
119        match self {
120            // Keyword tokens carry their spelling in the KEYWORDS table, so the
121            // spelling lives in one place; a new keyword variant makes this match
122            // non-exhaustive until it is added here too.
123            TokenKind::Pls
124            | TokenKind::Bork
125            | TokenKind::Bonk
126            | TokenKind::Bark
127            | TokenKind::Wow
128            | TokenKind::Such
129            | TokenKind::Much
130            | TokenKind::Many
131            | TokenKind::So
132            | TokenKind::Very
133            | TokenKind::Super
134            | TokenKind::If
135            | TokenKind::Elif
136            | TokenKind::Else
137            | TokenKind::For
138            | TokenKind::While
139            | TokenKind::In
140            | TokenKind::Return
141            | TokenKind::Continue
142            | TokenKind::And
143            | TokenKind::Or
144            | TokenKind::Not
145            | TokenKind::True
146            | TokenKind::False
147            | TokenKind::None
148            | TokenKind::Def
149            | TokenKind::Class
150            | TokenKind::Amaze => crate::keywords::keyword_spelling(self)
151                .expect("compiler bug: keyword token missing from KEYWORDS table")
152                .into(),
153            TokenKind::OhNo => "oh no".into(),
154            TokenKind::Ident(name) => format!("name '{name}'"),
155            TokenKind::Int(n) => format!("the number {n}"),
156            TokenKind::Float(f) => format!("the number {f}"),
157            TokenKind::Str(_) | TokenKind::StrInterp(_) => "a string".into(),
158            TokenKind::Plus => "+".into(),
159            TokenKind::Minus => "-".into(),
160            TokenKind::Star => "*".into(),
161            TokenKind::StarStar => "**".into(),
162            TokenKind::Slash => "/".into(),
163            TokenKind::SlashSlash => "//".into(),
164            TokenKind::Percent => "%".into(),
165            TokenKind::Amp => "&".into(),
166            TokenKind::Pipe => "|".into(),
167            TokenKind::Caret => "^".into(),
168            TokenKind::Tilde => "~".into(),
169            TokenKind::Shl => "<<".into(),
170            TokenKind::Shr => ">>".into(),
171            TokenKind::EqEq => "==".into(),
172            TokenKind::NotEq => "!=".into(),
173            TokenKind::Lt => "<".into(),
174            TokenKind::LtEq => "<=".into(),
175            TokenKind::Gt => ">".into(),
176            TokenKind::GtEq => ">=".into(),
177            TokenKind::Eq => "=".into(),
178            TokenKind::AugAssign(op) => format!("{}=", op.symbol()),
179            TokenKind::Colon => ":".into(),
180            TokenKind::Bang => "!".into(),
181            TokenKind::Comma => ",".into(),
182            TokenKind::Dot => ".".into(),
183            TokenKind::LParen => "(".into(),
184            TokenKind::RParen => ")".into(),
185            TokenKind::LBracket => "[".into(),
186            TokenKind::RBracket => "]".into(),
187            TokenKind::LBrace => "{".into(),
188            TokenKind::RBrace => "}".into(),
189            TokenKind::Newline => "the end of the line".into(),
190            TokenKind::Indent => "more indentation".into(),
191            TokenKind::Dedent => "less indentation".into(),
192            TokenKind::Eof => "the end of the script".into(),
193        }
194    }
195}
196
197#[cfg(test)]
198mod tests {
199    use super::*;
200    use crate::keywords::{keyword_spelling, lookup, KEYWORDS};
201
202    #[test]
203    fn every_keyword_round_trips_through_lookup_and_describe() {
204        for (spelling, kind) in KEYWORDS {
205            assert_eq!(lookup(spelling).as_ref(), Some(kind));
206            assert_eq!(keyword_spelling(kind), Some(*spelling));
207            assert_eq!(kind.describe(), *spelling);
208        }
209    }
210
211    #[test]
212    fn non_keyword_tokens_have_no_keyword_spelling() {
213        assert_eq!(keyword_spelling(&TokenKind::OhNo), None);
214        assert_eq!(keyword_spelling(&TokenKind::Plus), None);
215        assert_eq!(keyword_spelling(&TokenKind::Eof), None);
216    }
217
218    #[test]
219    fn ohno_describes_as_two_words() {
220        assert_eq!(TokenKind::OhNo.describe(), "oh no");
221    }
222}
doge_compiler/token.rs

doge_compiler/
token.rs
