use crate::cst::ListKind;
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum TokenKind {
Open(ListKind),
Close(char),
Atom,
Str,
Prefix,
Ws,
LineComment,
BlockComment,
DatumCommentStart,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Token {
pub kind: TokenKind,
pub text: String,
pub line: u32,
pub col: u32,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ParseError {
pub message: String,
pub line: u32,
pub col: u32,
}
impl std::fmt::Display for ParseError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}:{}: {}", self.line, self.col, self.message)
}
}
impl std::error::Error for ParseError {}
pub struct Lexer {
chars: Vec<char>,
pos: usize,
line: u32,
col: u32,
}
fn is_delimiter(c: char) -> bool {
c.is_whitespace() || matches!(c, '(' | ')' | '[' | ']' | '"' | ';')
}
impl Lexer {
pub fn new(src: &str) -> Self {
Lexer {
chars: src.chars().collect(),
pos: 0,
line: 1,
col: 1,
}
}
fn peek(&self, ahead: usize) -> Option<char> {
self.chars.get(self.pos + ahead).copied()
}
fn bump(&mut self) -> char {
let c = self.chars[self.pos];
self.pos += 1;
if c == '\n' {
self.line += 1;
self.col = 1;
} else {
self.col += 1;
}
c
}
fn error(&self, message: &str, line: u32, col: u32) -> ParseError {
ParseError {
message: message.to_string(),
line,
col,
}
}
pub fn next_token(&mut self) -> Result<Option<Token>, ParseError> {
let (line, col) = (self.line, self.col);
let Some(c) = self.peek(0) else {
return Ok(None);
};
let (kind, text) = match c {
_ if c.is_whitespace() => {
let mut text = String::new();
while self.peek(0).is_some_and(char::is_whitespace) {
text.push(self.bump());
}
(TokenKind::Ws, text)
}
';' => {
let mut text = String::new();
while self.peek(0).is_some_and(|c| c != '\n') {
text.push(self.bump());
}
(TokenKind::LineComment, text)
}
'(' => (TokenKind::Open(ListKind::Paren), self.bump().to_string()),
'[' => (TokenKind::Open(ListKind::Bracket), self.bump().to_string()),
')' | ']' => (TokenKind::Close(c), self.bump().to_string()),
'\'' | '`' => (TokenKind::Prefix, self.bump().to_string()),
',' => {
let mut text = self.bump().to_string();
if self.peek(0) == Some('@') {
text.push(self.bump());
}
(TokenKind::Prefix, text)
}
'"' => (TokenKind::Str, self.lex_string(line, col)?),
'#' => return self.lex_hash(line, col).map(Some),
_ => (TokenKind::Atom, self.lex_atom()),
};
Ok(Some(Token {
kind,
text,
line,
col,
}))
}
fn lex_string(&mut self, line: u32, col: u32) -> Result<String, ParseError> {
let mut text = self.bump().to_string();
loop {
match self.peek(0) {
None => return Err(self.error("unterminated string", line, col)),
Some('\\') => {
text.push(self.bump());
if self.peek(0).is_some() {
text.push(self.bump());
}
}
Some('"') => {
text.push(self.bump());
return Ok(text);
}
Some(_) => text.push(self.bump()),
}
}
}
fn lex_hash(&mut self, line: u32, col: u32) -> Result<Token, ParseError> {
let (kind, text) = match self.peek(1) {
Some('|') => (TokenKind::BlockComment, self.lex_block_comment(line, col)?),
Some(';') => {
let text: String = [self.bump(), self.bump()].iter().collect();
(TokenKind::DatumCommentStart, text)
}
Some('(') => {
let text: String = [self.bump(), self.bump()].iter().collect();
(TokenKind::Open(ListKind::Vector), text)
}
Some('\\') => {
let mut text: String = [self.bump(), self.bump()].iter().collect();
if self.peek(0).is_none() {
return Err(self.error("unterminated character literal", line, col));
}
text.push(self.bump());
while self.peek(0).is_some_and(|c| c.is_ascii_alphanumeric()) {
text.push(self.bump());
}
(TokenKind::Atom, text)
}
Some('\'') | Some('`') | Some('~') | Some('+') => {
let text: String = [self.bump(), self.bump()].iter().collect();
(TokenKind::Prefix, text)
}
Some(',') | Some('$') => {
let mut text: String = [self.bump(), self.bump()].iter().collect();
if self.peek(0) == Some('@') {
text.push(self.bump());
}
(TokenKind::Prefix, text)
}
_ => (TokenKind::Atom, self.lex_atom()),
};
Ok(Token {
kind,
text,
line,
col,
})
}
fn lex_block_comment(&mut self, line: u32, col: u32) -> Result<String, ParseError> {
let mut text: String = [self.bump(), self.bump()].iter().collect();
let mut depth = 1usize;
while depth > 0 {
match (self.peek(0), self.peek(1)) {
(Some('#'), Some('|')) => {
text.push(self.bump());
text.push(self.bump());
depth += 1;
}
(Some('|'), Some('#')) => {
text.push(self.bump());
text.push(self.bump());
depth -= 1;
}
(Some(_), _) => text.push(self.bump()),
(None, _) => return Err(self.error("unterminated block comment", line, col)),
}
}
Ok(text)
}
fn lex_atom(&mut self) -> String {
let mut text = String::new();
while self.peek(0).is_some_and(|c| !is_delimiter(c)) {
text.push(self.bump());
}
text
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::cst::ListKind;
fn kinds(src: &str) -> Vec<(TokenKind, String)> {
let mut lx = Lexer::new(src);
let mut out = Vec::new();
while let Some(t) = lx.next_token().unwrap() {
out.push((t.kind, t.text));
}
out
}
#[test]
fn lexes_atoms_strings_ws() {
assert_eq!(
kinds("(name 'guix)"),
vec![
(TokenKind::Open(ListKind::Paren), "(".into()),
(TokenKind::Atom, "name".into()),
(TokenKind::Ws, " ".into()),
(TokenKind::Prefix, "'".into()),
(TokenKind::Atom, "guix".into()),
(TokenKind::Close(')'), ")".into()),
]
);
}
#[test]
fn string_raw_with_escapes() {
let ks = kinds(r#""a\"b\\c""#);
assert_eq!(ks, vec![(TokenKind::Str, r#""a\"b\\c""#.into())]);
}
#[test]
fn line_comment_excludes_newline() {
let ks = kinds(";; hi\n(x)");
assert_eq!(ks[0], (TokenKind::LineComment, ";; hi".into()));
assert_eq!(ks[1], (TokenKind::Ws, "\n".into()));
}
#[test]
fn nested_block_comment() {
let ks = kinds("#| a #| b |# c |#x");
assert_eq!(ks[0], (TokenKind::BlockComment, "#| a #| b |# c |#".into()));
assert_eq!(ks[1], (TokenKind::Atom, "x".into()));
}
#[test]
fn hash_forms() {
assert_eq!(kinds("#t")[0], (TokenKind::Atom, "#t".into()));
assert_eq!(
kinds("#:use-module")[0],
(TokenKind::Atom, "#:use-module".into())
);
assert_eq!(kinds(r"#\(")[0], (TokenKind::Atom, r"#\(".into()));
assert_eq!(kinds(r"#\space")[0], (TokenKind::Atom, r"#\space".into()));
assert_eq!(
kinds("#(1)")[0],
(TokenKind::Open(ListKind::Vector), "#(".into())
);
assert_eq!(kinds("#;")[0], (TokenKind::DatumCommentStart, "#;".into()));
}
#[test]
fn gexp_prefixes() {
for p in [
"'", "`", ",", ",@", "#'", "#`", "#,", "#,@", "#~", "#$", "#$@", "#+",
] {
let src = format!("{p}x");
let ks = kinds(&src);
assert_eq!(ks[0], (TokenKind::Prefix, p.to_string()), "prefix {p}");
}
}
#[test]
fn unterminated_string_errors_with_position() {
let mut lx = Lexer::new("(x \"abc");
lx.next_token().unwrap();
lx.next_token().unwrap();
lx.next_token().unwrap();
let err = lx.next_token().unwrap_err();
assert_eq!(err.line, 1);
}
}