1use crate::cst::ListKind;
2
3#[derive(Debug, Clone, PartialEq, Eq)]
4pub enum TokenKind {
5 Open(ListKind),
6 Close(char),
7 Atom,
8 Str,
9 Prefix,
10 Ws,
11 LineComment,
12 BlockComment,
13 DatumCommentStart,
14}
15
16#[derive(Debug, Clone, PartialEq, Eq)]
17pub struct Token {
18 pub kind: TokenKind,
19 pub text: String,
20 pub line: u32,
21 pub col: u32,
22}
23
24#[derive(Debug, Clone, PartialEq, Eq)]
25pub struct ParseError {
26 pub message: String,
27 pub line: u32,
28 pub col: u32,
29}
30
31impl std::fmt::Display for ParseError {
32 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
33 write!(f, "{}:{}: {}", self.line, self.col, self.message)
34 }
35}
36
37impl std::error::Error for ParseError {}
38
39pub struct Lexer {
40 chars: Vec<char>,
41 pos: usize,
42 line: u32,
43 col: u32,
44}
45
46fn is_delimiter(c: char) -> bool {
48 c.is_whitespace() || matches!(c, '(' | ')' | '[' | ']' | '"' | ';')
49}
50
51impl Lexer {
52 pub fn new(src: &str) -> Self {
53 Lexer {
54 chars: src.chars().collect(),
55 pos: 0,
56 line: 1,
57 col: 1,
58 }
59 }
60
61 fn peek(&self, ahead: usize) -> Option<char> {
62 self.chars.get(self.pos + ahead).copied()
63 }
64
65 fn bump(&mut self) -> char {
66 let c = self.chars[self.pos];
67 self.pos += 1;
68 if c == '\n' {
69 self.line += 1;
70 self.col = 1;
71 } else {
72 self.col += 1;
73 }
74 c
75 }
76
77 fn error(&self, message: &str, line: u32, col: u32) -> ParseError {
78 ParseError {
79 message: message.to_string(),
80 line,
81 col,
82 }
83 }
84
85 pub fn next_token(&mut self) -> Result<Option<Token>, ParseError> {
86 let (line, col) = (self.line, self.col);
87 let Some(c) = self.peek(0) else {
88 return Ok(None);
89 };
90 let (kind, text) = match c {
91 _ if c.is_whitespace() => {
92 let mut text = String::new();
93 while self.peek(0).is_some_and(char::is_whitespace) {
94 text.push(self.bump());
95 }
96 (TokenKind::Ws, text)
97 }
98 ';' => {
99 let mut text = String::new();
100 while self.peek(0).is_some_and(|c| c != '\n') {
101 text.push(self.bump());
102 }
103 (TokenKind::LineComment, text)
104 }
105 '(' => (TokenKind::Open(ListKind::Paren), self.bump().to_string()),
106 '[' => (TokenKind::Open(ListKind::Bracket), self.bump().to_string()),
107 ')' | ']' => (TokenKind::Close(c), self.bump().to_string()),
108 '\'' | '`' => (TokenKind::Prefix, self.bump().to_string()),
109 ',' => {
110 let mut text = self.bump().to_string();
111 if self.peek(0) == Some('@') {
112 text.push(self.bump());
113 }
114 (TokenKind::Prefix, text)
115 }
116 '"' => (TokenKind::Str, self.lex_string(line, col)?),
117 '#' => return self.lex_hash(line, col).map(Some),
118 _ => (TokenKind::Atom, self.lex_atom()),
119 };
120 Ok(Some(Token {
121 kind,
122 text,
123 line,
124 col,
125 }))
126 }
127
128 fn lex_string(&mut self, line: u32, col: u32) -> Result<String, ParseError> {
129 let mut text = self.bump().to_string();
130 loop {
131 match self.peek(0) {
132 None => return Err(self.error("unterminated string", line, col)),
133 Some('\\') => {
134 text.push(self.bump());
135 if self.peek(0).is_some() {
136 text.push(self.bump());
137 }
138 }
139 Some('"') => {
140 text.push(self.bump());
141 return Ok(text);
142 }
143 Some(_) => text.push(self.bump()),
144 }
145 }
146 }
147
148 fn lex_hash(&mut self, line: u32, col: u32) -> Result<Token, ParseError> {
149 let (kind, text) = match self.peek(1) {
150 Some('|') => (TokenKind::BlockComment, self.lex_block_comment(line, col)?),
151 Some(';') => {
152 let text: String = [self.bump(), self.bump()].iter().collect();
153 (TokenKind::DatumCommentStart, text)
154 }
155 Some('(') => {
156 let text: String = [self.bump(), self.bump()].iter().collect();
157 (TokenKind::Open(ListKind::Vector), text)
158 }
159 Some('\\') => {
160 let mut text: String = [self.bump(), self.bump()].iter().collect();
161 if self.peek(0).is_none() {
162 return Err(self.error("unterminated character literal", line, col));
163 }
164 text.push(self.bump());
165 while self.peek(0).is_some_and(|c| c.is_ascii_alphanumeric()) {
166 text.push(self.bump());
167 }
168 (TokenKind::Atom, text)
169 }
170 Some('\'') | Some('`') | Some('~') | Some('+') => {
171 let text: String = [self.bump(), self.bump()].iter().collect();
172 (TokenKind::Prefix, text)
173 }
174 Some(',') | Some('$') => {
175 let mut text: String = [self.bump(), self.bump()].iter().collect();
176 if self.peek(0) == Some('@') {
177 text.push(self.bump());
178 }
179 (TokenKind::Prefix, text)
180 }
181 _ => (TokenKind::Atom, self.lex_atom()),
182 };
183 Ok(Token {
184 kind,
185 text,
186 line,
187 col,
188 })
189 }
190
191 fn lex_block_comment(&mut self, line: u32, col: u32) -> Result<String, ParseError> {
192 let mut text: String = [self.bump(), self.bump()].iter().collect();
193 let mut depth = 1usize;
194 while depth > 0 {
195 match (self.peek(0), self.peek(1)) {
196 (Some('#'), Some('|')) => {
197 text.push(self.bump());
198 text.push(self.bump());
199 depth += 1;
200 }
201 (Some('|'), Some('#')) => {
202 text.push(self.bump());
203 text.push(self.bump());
204 depth -= 1;
205 }
206 (Some(_), _) => text.push(self.bump()),
207 (None, _) => return Err(self.error("unterminated block comment", line, col)),
208 }
209 }
210 Ok(text)
211 }
212
213 fn lex_atom(&mut self) -> String {
214 let mut text = String::new();
215 while self.peek(0).is_some_and(|c| !is_delimiter(c)) {
216 text.push(self.bump());
217 }
218 text
219 }
220}
221
222#[cfg(test)]
223mod tests {
224 use super::*;
225 use crate::cst::ListKind;
226
227 fn kinds(src: &str) -> Vec<(TokenKind, String)> {
228 let mut lx = Lexer::new(src);
229 let mut out = Vec::new();
230 while let Some(t) = lx.next_token().unwrap() {
231 out.push((t.kind, t.text));
232 }
233 out
234 }
235
236 #[test]
237 fn lexes_atoms_strings_ws() {
238 assert_eq!(
239 kinds("(name 'guix)"),
240 vec![
241 (TokenKind::Open(ListKind::Paren), "(".into()),
242 (TokenKind::Atom, "name".into()),
243 (TokenKind::Ws, " ".into()),
244 (TokenKind::Prefix, "'".into()),
245 (TokenKind::Atom, "guix".into()),
246 (TokenKind::Close(')'), ")".into()),
247 ]
248 );
249 }
250
251 #[test]
252 fn string_raw_with_escapes() {
253 let ks = kinds(r#""a\"b\\c""#);
254 assert_eq!(ks, vec![(TokenKind::Str, r#""a\"b\\c""#.into())]);
255 }
256
257 #[test]
258 fn line_comment_excludes_newline() {
259 let ks = kinds(";; hi\n(x)");
260 assert_eq!(ks[0], (TokenKind::LineComment, ";; hi".into()));
261 assert_eq!(ks[1], (TokenKind::Ws, "\n".into()));
262 }
263
264 #[test]
265 fn nested_block_comment() {
266 let ks = kinds("#| a #| b |# c |#x");
267 assert_eq!(ks[0], (TokenKind::BlockComment, "#| a #| b |# c |#".into()));
268 assert_eq!(ks[1], (TokenKind::Atom, "x".into()));
269 }
270
271 #[test]
272 fn hash_forms() {
273 assert_eq!(kinds("#t")[0], (TokenKind::Atom, "#t".into()));
274 assert_eq!(
275 kinds("#:use-module")[0],
276 (TokenKind::Atom, "#:use-module".into())
277 );
278 assert_eq!(kinds(r"#\(")[0], (TokenKind::Atom, r"#\(".into()));
279 assert_eq!(kinds(r"#\space")[0], (TokenKind::Atom, r"#\space".into()));
280 assert_eq!(
281 kinds("#(1)")[0],
282 (TokenKind::Open(ListKind::Vector), "#(".into())
283 );
284 assert_eq!(kinds("#;")[0], (TokenKind::DatumCommentStart, "#;".into()));
285 }
286
287 #[test]
288 fn gexp_prefixes() {
289 for p in [
290 "'", "`", ",", ",@", "#'", "#`", "#,", "#,@", "#~", "#$", "#$@", "#+",
291 ] {
292 let src = format!("{p}x");
293 let ks = kinds(&src);
294 assert_eq!(ks[0], (TokenKind::Prefix, p.to_string()), "prefix {p}");
295 }
296 }
297
298 #[test]
299 fn unterminated_string_errors_with_position() {
300 let mut lx = Lexer::new("(x \"abc");
301 lx.next_token().unwrap();
302 lx.next_token().unwrap();
303 lx.next_token().unwrap();
304 let err = lx.next_token().unwrap_err();
305 assert_eq!(err.line, 1);
306 }
307}