use crate::ast::{DrawOp, LineStyle, LinkMarker, LinkOp};
use crate::error::Error;
use crate::span::Span;
#[derive(Debug, Clone, PartialEq)]
pub enum TokKind {
Ident(String),
String(String),
Number(f64),
Percent(f64), Hash(String),
RawCssVar(String), Expr(String),
Pipe, Colon, DColon, Dot, Amp, Semi, Comma, LBrace,
RBrace,
LParen,
RParen,
LBracket,
RBracket,
LinkOp(LinkOp),
DrawOp(DrawOp),
Newline,
}
#[derive(Debug, Clone)]
pub struct Token {
pub kind: TokKind,
pub span: Span,
}
pub fn lex(src: &str) -> Result<Vec<Token>, Error> {
let mut lexer = Lexer {
src,
bytes: src.as_bytes(),
i: 0,
paren_depth: 0,
tokens: Vec::new(),
};
lexer.run()?;
Ok(lexer.tokens)
}
struct Lexer<'a> {
src: &'a str,
bytes: &'a [u8],
i: usize,
paren_depth: usize,
tokens: Vec<Token>,
}
impl<'a> Lexer<'a> {
fn run(&mut self) -> Result<(), Error> {
while self.i < self.bytes.len() {
let c = self.bytes[self.i];
match c {
b' ' | b'\t' | b'\r' => self.i += 1,
b'\n' => self.handle_newline(),
b'/' if self.peek(1) == Some(b'/') => self.skip_line_comment(),
b'{' => self.push_punct(TokKind::LBrace, 1),
b'}' => self.push_punct(TokKind::RBrace, 1),
b'(' => {
let glued_call = self.i > 0 && is_ident_continue(self.bytes[self.i - 1]);
let rest = &self.bytes[self.i..];
if !glued_call && rest.starts_with(b"(-)") {
self.push_punct(TokKind::DrawOp(DrawOp::Linear), 3);
} else if !glued_call && rest.starts_with(b"(o)") {
self.push_punct(TokKind::DrawOp(DrawOp::Round), 3);
} else if !glued_call && rest.starts_with(b"(<)") {
self.push_punct(TokKind::DrawOp(DrawOp::Angle), 3);
} else if !glued_call && rest.starts_with(b"(>)") {
return Err(Error::at(
Span::new(self.i, self.i + 3),
"'(>)' is reserved — the angle op is '(<)'",
));
} else {
self.paren_depth += 1;
self.push_punct(TokKind::LParen, 1);
}
}
b')' => {
self.paren_depth = self.paren_depth.saturating_sub(1);
self.push_punct(TokKind::RParen, 1);
}
b'[' => self.push_punct(TokKind::LBracket, 1),
b']' => self.push_punct(TokKind::RBracket, 1),
b'|' => self.push_punct(TokKind::Pipe, 1),
b':' if self.peek(1) == Some(b':') => self.push_punct(TokKind::DColon, 2),
b':' => self.push_punct(TokKind::Colon, 1),
b';' => self.push_punct(TokKind::Semi, 1),
b',' => self.push_punct(TokKind::Comma, 1),
b'&' => self.push_punct(TokKind::Amp, 1),
b'"' => self.lex_string()?,
b'\'' => {
return Err(Error::at(
Span::new(self.i, self.i + 1),
"single quotes are not strings — use \"…\"",
));
}
b'`' => self.lex_expr()?,
b'#' => self.lex_hash()?,
b'.' => {
if self.peek(1).is_some_and(|c| c.is_ascii_digit()) {
self.lex_number()?;
} else {
self.push_punct(TokKind::Dot, 1);
}
}
b'-' if self.peek(1) == Some(b'-') && self.peek(2).is_some_and(is_ident_start) => {
self.lex_raw_css_var()?;
}
b'-' if self.peek(1).is_some_and(|c| c.is_ascii_digit())
|| (self.peek(1) == Some(b'.')
&& self.peek(2).is_some_and(|c| c.is_ascii_digit())) =>
{
self.lex_number()?;
}
b'+' if self.peek(1).is_some_and(|c| c.is_ascii_digit())
|| (self.peek(1) == Some(b'.')
&& self.peek(2).is_some_and(|c| c.is_ascii_digit())) =>
{
self.lex_number()?;
}
b'-' | b'~' | b'<' | b'>' | b'+' => self.lex_link_op()?,
b'*' if self.peek(1).is_some_and(is_link_line_start) => self.lex_link_op()?,
d if d.is_ascii_digit() => self.lex_number()?,
c if is_ident_start(c) => self.lex_ident(),
_ => {
return Err(Error::at(
Span::new(self.i, self.i + 1),
format!("unexpected character {:?}", c as char),
));
}
}
}
Ok(())
}
fn peek(&self, n: usize) -> Option<u8> {
self.bytes.get(self.i + n).copied()
}
fn push_punct(&mut self, kind: TokKind, len: usize) {
let span = Span::new(self.i, self.i + len);
self.tokens.push(Token { kind, span });
self.i += len;
}
fn handle_newline(&mut self) {
let start = self.i;
self.i += 1;
while self.i < self.bytes.len() {
let c = self.bytes[self.i];
if c == b' ' || c == b'\t' || c == b'\r' || c == b'\n' {
self.i += 1;
} else {
break;
}
}
if self.paren_depth == 0 {
self.tokens.push(Token {
kind: TokKind::Newline,
span: Span::new(start, start + 1),
});
}
}
fn skip_line_comment(&mut self) {
while self.i < self.bytes.len() && self.bytes[self.i] != b'\n' {
self.i += 1;
}
}
fn lex_string(&mut self) -> Result<(), Error> {
let start = self.i;
self.i += 1; let mut value = String::new();
while self.i < self.bytes.len() {
let b = self.bytes[self.i];
if b == b'"' {
self.i += 1;
self.tokens.push(Token {
kind: TokKind::String(value.trim().to_string()),
span: Span::new(start, self.i),
});
return Ok(());
}
if b == b'\\' {
let esc_start = self.i;
self.i += 1;
let next = self.bytes.get(self.i).copied().ok_or_else(|| {
Error::at(Span::new(esc_start, self.i), "unterminated escape sequence")
})?;
match next {
b'"' => value.push('"'),
b'\\' => value.push('\\'),
b'n' => value.push('\n'),
b't' => value.push('\t'),
other => {
return Err(Error::at(
Span::new(esc_start, self.i + 1),
format!("invalid escape sequence '\\{}'", other as char),
));
}
}
self.i += 1;
continue;
}
let ch = self.src[self.i..].chars().next().expect("non-empty utf-8");
value.push(ch);
self.i += ch.len_utf8();
}
Err(Error::at(
Span::new(start, self.i),
"unterminated string literal",
))
}
fn lex_expr(&mut self) -> Result<(), Error> {
let start = self.i;
self.i += 1; let body_start = self.i;
while self.i < self.bytes.len() && self.bytes[self.i] != b'`' {
self.i += 1;
}
if self.i >= self.bytes.len() {
return Err(Error::at(
Span::new(start, self.i),
"unterminated `…` expression",
));
}
let body = self.src[body_start..self.i].to_string();
self.i += 1; self.tokens.push(Token {
kind: TokKind::Expr(body),
span: Span::new(start, self.i),
});
Ok(())
}
fn lex_hash(&mut self) -> Result<(), Error> {
let start = self.i;
self.i += 1; let run_start = self.i;
while self.i < self.bytes.len() && is_ident_continue(self.bytes[self.i]) {
self.i += 1;
}
let run = self.src[run_start..self.i].to_string();
self.tokens.push(Token {
kind: TokKind::Hash(run),
span: Span::new(start, self.i),
});
Ok(())
}
fn lex_raw_css_var(&mut self) -> Result<(), Error> {
let start = self.i;
self.i += 2; let name_start = self.i;
while self.i < self.bytes.len() && is_ident_continue(self.bytes[self.i]) {
self.i += 1;
}
let name = self.src[name_start..self.i].to_string();
self.tokens.push(Token {
kind: TokKind::RawCssVar(name),
span: Span::new(start, self.i),
});
Ok(())
}
fn lex_number(&mut self) -> Result<(), Error> {
let start = self.i;
if matches!(self.bytes[self.i], b'+' | b'-') {
self.i += 1;
}
let mut saw_digit = false;
while self.i < self.bytes.len() && self.bytes[self.i].is_ascii_digit() {
self.i += 1;
saw_digit = true;
}
if self.i < self.bytes.len()
&& self.bytes[self.i] == b'.'
&& self.peek(1).is_some_and(|c| c.is_ascii_digit())
{
self.i += 1; while self.i < self.bytes.len() && self.bytes[self.i].is_ascii_digit() {
self.i += 1;
saw_digit = true;
}
}
if !saw_digit {
return Err(Error::at(
Span::new(start, self.i),
"invalid number literal",
));
}
let text = &self.src[start..self.i];
let value: f64 = text.parse().map_err(|_| {
Error::at(
Span::new(start, self.i),
format!("invalid number literal '{}'", text),
)
})?;
let kind = if self.i < self.bytes.len() && self.bytes[self.i] == b'%' {
self.i += 1;
TokKind::Percent(value)
} else {
TokKind::Number(value)
};
self.tokens.push(Token {
kind,
span: Span::new(start, self.i),
});
Ok(())
}
fn lex_ident(&mut self) {
let start = self.i;
while self.i < self.bytes.len() && is_ident_continue(self.bytes[self.i]) {
self.i += 1;
}
let name = self.src[start..self.i].to_string();
self.tokens.push(Token {
kind: TokKind::Ident(name),
span: Span::new(start, self.i),
});
}
fn lex_link_op(&mut self) -> Result<(), Error> {
let start_i = self.i;
let mut p = self.i;
let start = self.consume_marker(&mut p, MarkerSide::Start)?;
let line = match self.consume_line(&mut p) {
Some(l) => l,
None => {
return Err(Error::at(
Span::new(start_i, p.max(start_i + 1)),
format!(
"expected link-op line after '{}'",
&self.src[start_i..p.max(start_i + 1)]
),
));
}
};
let end = self.consume_marker(&mut p, MarkerSide::End)?;
let op = LinkOp { line, start, end };
let span = Span::new(start_i, p);
self.tokens.push(Token {
kind: TokKind::LinkOp(op),
span,
});
self.i = p;
Ok(())
}
fn consume_marker(&self, p: &mut usize, side: MarkerSide) -> Result<LinkMarker, Error> {
if self.bytes.get(*p) == Some(&b'<') && self.bytes.get(*p + 1) == Some(&b'>') {
*p += 2;
return Ok(LinkMarker::Diamond);
}
let c = match self.bytes.get(*p) {
Some(&c) => c,
None => return Ok(LinkMarker::None),
};
let next = self.bytes.get(*p + 1).copied();
match side {
MarkerSide::End if c == b'+' || c == b'o' => {
let hit = match (c, next) {
(b'+', Some(b'+')) => Some((LinkMarker::ExactlyOne, 2)),
(b'+', Some(b'<')) => Some((LinkMarker::OneOrMany, 2)),
(b'+', _) => Some((LinkMarker::One, 1)),
(b'o', Some(b'+')) => Some((LinkMarker::ZeroOrOne, 2)),
(b'o', Some(b'<')) => Some((LinkMarker::ZeroOrMany, 2)),
_ => None, };
return self.finish_cardinality(p, hit);
}
MarkerSide::Start if c == b'+' || c == b'>' => {
let hit = match (c, next) {
(b'+', Some(b'+')) => Some((LinkMarker::ExactlyOne, 2)),
(b'+', Some(b'o')) => Some((LinkMarker::ZeroOrOne, 2)),
(b'+', _) => Some((LinkMarker::One, 1)),
(b'>', Some(b'+')) => Some((LinkMarker::OneOrMany, 2)),
(b'>', Some(b'o')) => Some((LinkMarker::ZeroOrMany, 2)),
(b'>', _) => Some((LinkMarker::Crow, 1)), _ => None, };
return self.finish_cardinality(p, hit);
}
_ => {}
}
let marker = match (c, side) {
(b'<', MarkerSide::Start) => LinkMarker::Arrow,
(b'<', MarkerSide::End) => LinkMarker::Crow,
(b'>', MarkerSide::End) => LinkMarker::Arrow,
(b'*', _) => LinkMarker::Dot,
_ => return Ok(LinkMarker::None),
};
*p += 1;
Ok(marker)
}
fn finish_cardinality(
&self,
p: &mut usize,
hit: Option<(LinkMarker, usize)>,
) -> Result<LinkMarker, Error> {
match hit {
Some((m, n)) => {
*p += n;
Ok(m)
}
None => Err(Error::at(
Span::new(p.saturating_sub(1), *p + 1),
"'-o' needs a max glyph — write '-o<', '-o+', or 'marker-end: circle'",
)),
}
}
fn consume_line(&self, p: &mut usize) -> Option<LineStyle> {
let rest = self.bytes.get(*p..)?;
if rest.starts_with(b"---") {
*p += 3;
return Some(LineStyle::Dotted);
}
if rest.starts_with(b"--") {
if rest.get(2).is_some_and(|&c| is_ident_start(c)) {
return None;
}
*p += 2;
return Some(LineStyle::Dashed);
}
match rest.first()? {
b'-' => {
*p += 1;
Some(LineStyle::Solid)
}
b'~' => {
*p += 1;
Some(LineStyle::Wavy)
}
_ => None,
}
}
}
#[derive(Clone, Copy)]
enum MarkerSide {
Start,
End,
}
fn is_link_line_start(c: u8) -> bool {
matches!(c, b'-' | b'~')
}
fn is_ident_start(c: u8) -> bool {
c.is_ascii_alphabetic() || c == b'_'
}
fn is_ident_continue(c: u8) -> bool {
c.is_ascii_alphanumeric() || c == b'_' || c == b'-'
}
#[cfg(test)]
mod tests {
use super::*;
use crate::ast::{LineStyle, LinkMarker};
fn kinds(src: &str) -> Vec<TokKind> {
lex(src)
.expect("lex ok")
.into_iter()
.map(|t| t.kind)
.filter(|k| !matches!(k, TokKind::Newline))
.collect()
}
fn line_of(src: &str) -> LineStyle {
match &kinds(src)[..] {
[TokKind::LinkOp(op)] => op.line,
other => panic!("expected one link op, got {other:?}"),
}
}
#[test]
fn hash_lexes_raw_run_for_colours_and_ids() {
assert_eq!(kinds("#fff"), vec![TokKind::Hash("fff".into())]);
assert_eq!(kinds("#ffaa00cc"), vec![TokKind::Hash("ffaa00cc".into())]);
assert_eq!(kinds("#cat"), vec![TokKind::Hash("cat".into())]);
assert_eq!(
kinds("#load-balancer"),
vec![TokKind::Hash("load-balancer".into())]
);
}
#[test]
fn link_lines_grow_more_broken() {
assert_eq!(line_of("->"), LineStyle::Solid);
assert_eq!(line_of("-->"), LineStyle::Dashed);
assert_eq!(line_of("--->"), LineStyle::Dotted);
assert_eq!(line_of("~>"), LineStyle::Wavy);
}
#[test]
fn arrow_marker_sits_at_the_end() {
let TokKind::LinkOp(op) = &kinds("->")[0] else {
panic!("expected a link op");
};
assert_eq!(op.start, LinkMarker::None);
assert_eq!(op.end, LinkMarker::Arrow);
}
#[test]
fn endpoint_side_is_ident_colon_ident() {
assert_eq!(
kinds("a:left"),
vec![
TokKind::Ident("a".into()),
TokKind::Colon,
TokKind::Ident("left".into()),
]
);
}
#[test]
fn css_var_beats_the_dashed_line() {
assert_eq!(kinds("--brand"), vec![TokKind::RawCssVar("brand".into())]);
}
#[test]
fn double_dot_is_two_dots_not_a_line() {
assert_eq!(kinds(".."), vec![TokKind::Dot, TokKind::Dot]);
}
#[test]
fn string_values_are_trimmed() {
assert_eq!(kinds("\" ABC \""), vec![TokKind::String("ABC".into())]);
assert_eq!(kinds("\"a b\""), vec![TokKind::String("a b".into())]);
}
#[test]
fn measuring_ops_lex_free_standing_only() {
use crate::ast::DrawOp;
assert_eq!(kinds("(-)"), vec![TokKind::DrawOp(DrawOp::Linear)]);
assert_eq!(kinds("(o)"), vec![TokKind::DrawOp(DrawOp::Round)]);
assert_eq!(kinds("(<)"), vec![TokKind::DrawOp(DrawOp::Angle)]);
assert_eq!(
kinds("foo(o)"),
vec![
TokKind::Ident("foo".into()),
TokKind::LParen,
TokKind::Ident("o".into()),
TokKind::RParen,
]
);
assert_eq!(
kinds("move(-90, 0)"),
vec![
TokKind::Ident("move".into()),
TokKind::LParen,
TokKind::Number(-90.0),
TokKind::Comma,
TokKind::Number(0.0),
TokKind::RParen,
]
);
assert_eq!(kinds("(-90)")[0], TokKind::LParen);
}
#[test]
fn reversed_angle_op_is_reserved() {
let err = lex("a (>) b").expect_err("(>) is reserved");
assert!(err.message.contains("the angle op is '(<)'"), "{err:?}");
}
#[test]
fn cardinality_markers_compose_min_max_on_both_sides() {
let op_of = |src: &str| match &kinds(src)[..] {
[TokKind::Ident(_), TokKind::LinkOp(op), TokKind::Ident(_)] => *op,
other => panic!("expected `a OP b`, got {other:?}"),
};
assert_eq!(op_of("a -+ b").end, LinkMarker::One);
assert_eq!(op_of("a -< b").end, LinkMarker::Crow);
assert_eq!(op_of("a -++ b").end, LinkMarker::ExactlyOne);
assert_eq!(op_of("a -o+ b").end, LinkMarker::ZeroOrOne);
assert_eq!(op_of("a -+< b").end, LinkMarker::OneOrMany);
assert_eq!(op_of("a -o< b").end, LinkMarker::ZeroOrMany);
assert_eq!(op_of("a +- b").start, LinkMarker::One);
assert_eq!(op_of("a >- b").start, LinkMarker::Crow);
assert_eq!(op_of("a ++- b").start, LinkMarker::ExactlyOne);
assert_eq!(op_of("a +o- b").start, LinkMarker::ZeroOrOne);
assert_eq!(op_of("a >+- b").start, LinkMarker::OneOrMany);
assert_eq!(op_of("a >o- b").start, LinkMarker::ZeroOrMany);
let both = op_of("a +-< b");
assert_eq!((both.start, both.end), (LinkMarker::One, LinkMarker::Crow));
assert_eq!(op_of("a +-+ b").start, LinkMarker::One);
assert_eq!(op_of("a +-+ b").end, LinkMarker::One);
assert_eq!(kinds("+5"), vec![TokKind::Number(5.0)]);
let err = lex("a -o b").expect_err("bare -o is an error");
assert!(err.message.contains("'-o' needs a max glyph"), "{err:?}");
}
}