libdusk/
token.rs

1use std::ffi::CString;
2
3use string_interner::DefaultSymbol as Sym;
4
5use dusk_dire::source_info::SourceRange;
6
7#[derive(Debug, PartialEq, Clone)]
8pub enum TokenKind {
9    Ident(Sym),
10
11    IntLit(u64),
12    DecLit(f64),
13    StrLit(CString),
14    CharLit(i8),
15
16    // Special
17    Eof,
18    Whitespace,
19    Newline,
20    SingleLineComment,
21    MultiLineComment,
22
23    // Keywords
24    Fn,
25    Return,
26    True,
27    False,
28    If,
29    Else,
30    While,
31    For,
32    Break,
33    Continue,
34    In,
35    Switch,
36    As,
37    Struct,
38    Enum,
39    Do,
40    Mut,
41    Module,
42    ExternModule,
43    Import,
44    DebugMark,
45
46    // Symbols
47    Colon,
48    Semicolon,
49    Comma,
50    LeftParen,
51    RightParen,
52    DoubleDot,
53    Dot,
54    OpenCurly,
55    CloseCurly,
56    OpenSquareBracket,
57    CloseSquareBracket,
58    AtSign,
59    ReturnArrow,
60
61    // Operators
62    Tilde,
63    Caret,
64    LeftShift,
65    RightShift,
66    AddAssign,
67    SubAssign,
68    MultAssign,
69    DivAssign,
70    ModAssign,
71    BitwiseOrAssign,
72    BitwiseAndAssign,
73    XorAssign,
74    LeftShiftAssign,
75    RightShiftAssign,
76    Add,
77    Sub,
78    Asterisk,
79    Div,
80    Mod,
81    Equal,
82    NotEqual,
83    Lte,
84    Lt,
85    Gte,
86    GT,
87    LogicalOr,
88    LogicalAnd,
89    LogicalNot,
90    Assign,
91    Ampersand,
92    Pipe,
93}
94
95#[derive(Debug)]
96pub struct TokenVec {
97    pub kinds: Vec<TokenKind>,
98    pub ranges: Vec<SourceRange>,
99}
100
101#[derive(Debug)]
102pub struct Token<'src> {
103    pub kind: &'src TokenKind,
104    pub range: SourceRange,
105}
106
107impl Default for TokenVec {
108    fn default() -> Self {
109        Self::new()
110    }
111}
112
113impl TokenVec {
114    pub fn new() -> Self {
115        Self {
116            kinds: Vec::new(),
117            ranges: Vec::new(),
118        }
119    }
120
121    pub fn push(&mut self, kind: TokenKind, range: SourceRange) {
122        self.kinds.push(kind);
123        self.ranges.push(range);
124    }
125
126    pub fn at(&self, i: usize) -> Token {
127        let i = std::cmp::min(i, self.kinds.len()-1);
128        Token {
129            kind: &self.kinds[i],
130            range: self.ranges[i],
131        }
132    }
133
134    pub fn len(&self) -> usize { self.kinds.len() }
135}
136
137impl TokenKind {
138    pub fn is_insignificant(&self) -> bool {
139        use TokenKind::*;
140        matches!(self, Whitespace | SingleLineComment | MultiLineComment | Newline)
141    }
142
143    pub fn is_significant(&self) -> bool { !self.is_insignificant() }
144
145    pub fn could_begin_expression(&self) -> bool {
146        use TokenKind::*;
147        !matches!(
148            self,
149            Eof | Whitespace | Newline | SingleLineComment | MultiLineComment | Else | As |
150            Mut | Comma | RightParen | OpenCurly | CloseCurly | AddAssign | SubAssign |
151            MultAssign | DivAssign | ModAssign | BitwiseOrAssign | BitwiseAndAssign |
152            Div | Mod | Equal | NotEqual | Lte | Lt | Gte | GT | LogicalOr | LogicalAnd | Assign |
153            Pipe | OpenSquareBracket | CloseSquareBracket
154        )
155    }
156
157    pub fn could_begin_struct_literal_field(&self) -> bool {
158        matches!(self, TokenKind::Ident(_))
159    }
160
161    pub fn could_begin_struct_field(&self) -> bool {
162        // attributes on struct fields aren't supported yet, but probably will be.
163        matches!(self, TokenKind::Ident(_) | TokenKind::AtSign)
164    }
165
166    pub fn could_begin_parameter(&self) -> bool {
167        // attributes on parameters aren't supported yet, but probably will be.
168        matches!(self, TokenKind::Ident(_) | TokenKind::AtSign)
169    }
170
171    pub fn could_begin_variant_decl(&self) -> bool {
172        // attributes on enum variants aren't supported yet, but probably will be.
173        matches!(self, TokenKind::Ident(_) | TokenKind::AtSign)
174    }
175
176    pub fn could_begin_pattern(&self) -> bool {
177        matches!(self, TokenKind::Dot | TokenKind::Ident(_) | TokenKind::IntLit(_))
178    }
179
180    pub fn could_begin_statement(&self) -> bool {
181        self.could_begin_expression() || 
182            self.could_begin_pattern() || 
183            matches!(self, TokenKind::AtSign | TokenKind::Ident(_) | TokenKind::Mut | TokenKind::Fn)
184    }
185
186    pub fn could_begin_generic_parameter(&self) -> bool {
187        matches!(self, TokenKind::Ident(_))
188    }
189
190    pub fn pretty_print_separator(&self) -> Option<&'static str> {
191        match self {
192            TokenKind::Comma => Some(","),
193            TokenKind::Semicolon => Some(";"),
194            _ => None,
195        }
196    }
197}
libdusk/token.rs

libdusk/
token.rs
