use crate::ast::Color;
use crate::error::PvgError;
#[derive(Debug, Clone, PartialEq)]
pub enum TokenKind {
Indent,
Dedent,
Newline,
Eof,
Number(f64),
String(String),
Color(Color),
Ident(String),
Pvg,
Canvas,
Background,
Snap,
Filter,
Param,
Pattern,
Sprite,
Spline,
Palette,
Data,
Row,
Set,
Def,
Return,
For,
From,
To,
Step,
While,
If,
Else,
Seed,
Circle,
Ellipse,
Rectangle,
Line,
Polygon,
Path,
Text,
Group,
Clip,
Center,
Radius,
Pos,
Size,
Points,
Content,
Font,
Align,
Fill,
Stroke,
Width,
Opacity,
Rot,
Scale,
Cap,
Join,
Miter,
Dash,
Blend,
Blur,
Shadow,
Glow,
Linear,
Radial,
Angular,
Stop,
Start,
Quad,
Curve,
Arc,
Close,
LBracket,
RBracket,
LParen,
RParen,
Comma,
Question,
Colon,
Plus,
Minus,
Star,
Slash,
Percent,
Caret,
Equal,
EqualEqual,
NotEqual,
Less,
LessEqual,
Greater,
GreaterEqual,
And,
Or,
Not,
}
#[derive(Debug, Clone, PartialEq)]
pub struct Token {
pub kind: TokenKind,
pub line: usize,
pub col: usize,
}
pub struct Lexer<'a> {
source: &'a str,
indent_stack: Vec<usize>,
}
impl<'a> Lexer<'a> {
pub fn new(source: &'a str) -> Self {
Self {
source,
indent_stack: vec![0],
}
}
pub fn tokenize_all(&mut self) -> Result<Vec<Token>, PvgError> {
let lines: Vec<&str> = self.source.lines().collect();
let mut tokens = Vec::new();
let mut li: usize = 0;
while li < lines.len() {
let line_num = li + 1;
let raw_line = lines[li];
let bytes = raw_line.as_bytes();
let mut spaces = 0;
while spaces < bytes.len() && bytes[spaces] == b' ' {
spaces += 1;
}
if spaces < bytes.len() && bytes[spaces] == b'\t' {
return Err(PvgError::lex(
line_num,
spaces + 1,
"Tabs are forbidden. Use 2 spaces for indentation.",
));
}
if spaces == bytes.len() || bytes[spaces] == b'#' {
li += 1;
continue;
}
let current_indent = *self.indent_stack.last().unwrap();
if spaces > current_indent {
self.indent_stack.push(spaces);
tokens.push(Token {
kind: TokenKind::Indent,
line: line_num,
col: spaces + 1,
});
} else if spaces < current_indent {
while let Some(&last) = self.indent_stack.last() {
if spaces < last {
self.indent_stack.pop();
tokens.push(Token {
kind: TokenKind::Dedent,
line: line_num,
col: spaces + 1,
});
} else if spaces == last {
break;
} else {
return Err(PvgError::lex(
line_num,
spaces + 1,
"Inconsistent indentation level.",
));
}
}
}
let content = &raw_line[spaces..];
let extra = if triple_opener_before_comment(content) {
self.tokenize_line_with_triple(
content, &lines, &mut li, line_num, spaces + 1, spaces, &mut tokens,
)?
} else {
self.tokenize_line(content, line_num, spaces + 1, &mut tokens)?;
0
};
tokens.push(Token {
kind: TokenKind::Newline,
line: line_num,
col: raw_line.len() + 1,
});
li += 1 + extra;
}
let last_line = lines.len().max(1);
while self.indent_stack.len() > 1 {
self.indent_stack.pop();
tokens.push(Token {
kind: TokenKind::Dedent,
line: last_line,
col: 1,
});
}
tokens.push(Token {
kind: TokenKind::Eof,
line: last_line,
col: 1,
});
Ok(tokens)
}
#[allow(clippy::too_many_arguments)]
fn tokenize_line_with_triple(
&self,
content: &str,
lines: &[&str],
li: &mut usize,
line_num: usize,
col_offset: usize,
base_spaces: usize,
tokens: &mut Vec<Token>,
) -> Result<usize, PvgError> {
let open = content.find("\"\"\"").unwrap();
let (prefix, rest) = content.split_at(open);
if !prefix.trim().is_empty() {
self.tokenize_line(prefix, line_num, col_offset, tokens)?;
}
let open_col = col_offset + open;
let after_open = &rest[3..];
if let Some(close) = after_open.find("\"\"\"") {
tokens.push(Token {
kind: TokenKind::String(after_open[..close].to_string()),
line: line_num,
col: open_col,
});
let tail = &after_open[close + 3..];
if !tail.trim().is_empty() {
self.tokenize_line(tail, line_num, open_col + 3 + close + 3, tokens)?;
}
return Ok(0);
}
{
let tail = after_open.trim_start();
if !(tail.is_empty() || tail.starts_with('#')) {
return Err(PvgError::lex(
line_num,
open_col,
"Multi-line \"\"\" strings must start at the end of the line (e.g. `data \"\"\"`).",
));
}
}
let mut body: Vec<String> = Vec::new();
let mut extra = 0usize;
loop {
let nli = *li + 1 + extra;
if nli >= lines.len() {
return Err(PvgError::lex(line_num, open_col, "Unclosed triple-quoted string."));
}
let l = lines[nli];
let lb = l.as_bytes();
let mut sp = 0;
while sp < lb.len() && lb[sp] == b' ' {
sp += 1;
}
if l[sp..].starts_with("\"\"\"") {
let tail = &l[sp + 3..];
let tail_trim = tail.trim_start();
if !(tail_trim.is_empty() || tail_trim.starts_with('#')) {
return Err(PvgError::lex(
nli + 1,
sp + 4,
"Unexpected content after closing \"\"\".",
));
}
extra += 1; break;
}
let mut cut = 0;
while cut < base_spaces && cut < lb.len() && lb[cut] == b' ' {
cut += 1;
}
body.push(l[cut..].to_string());
extra += 1;
}
tokens.push(Token {
kind: TokenKind::String(body.join("\n")),
line: line_num,
col: open_col,
});
Ok(extra)
}
fn tokenize_line(
&self,
text: &str,
line_num: usize,
col_offset: usize,
tokens: &mut Vec<Token>,
) -> Result<(), PvgError> {
let bytes = text.as_bytes();
let len = bytes.len();
let mut i = 0;
while i < len {
let b = bytes[i];
if b == b' ' || b == b'\t' || b == b'\r' {
i += 1;
continue;
}
let col = col_offset + i;
if b == b'#' {
let mut hex_end = i + 1;
while hex_end < len && bytes[hex_end].is_ascii_hexdigit() {
hex_end += 1;
}
let hex_len = hex_end - (i + 1);
if hex_len == 3 || hex_len == 6 || hex_len == 8 {
let is_delim = hex_end == len
|| bytes[hex_end].is_ascii_whitespace()
|| bytes[hex_end] == b']'
|| bytes[hex_end] == b')'
|| bytes[hex_end] == b','
|| bytes[hex_end] == b':';
if is_delim {
let hex_str = std::str::from_utf8(&bytes[i..hex_end]).unwrap();
if let Some(col_val) = Color::from_hex(hex_str) {
tokens.push(Token {
kind: TokenKind::Color(col_val),
line: line_num,
col,
});
i = hex_end;
continue;
}
}
}
break;
}
if b == b'[' { tokens.push(Token { kind: TokenKind::LBracket, line: line_num, col }); i += 1; continue; }
if b == b']' { tokens.push(Token { kind: TokenKind::RBracket, line: line_num, col }); i += 1; continue; }
if b == b'(' { tokens.push(Token { kind: TokenKind::LParen, line: line_num, col }); i += 1; continue; }
if b == b')' { tokens.push(Token { kind: TokenKind::RParen, line: line_num, col }); i += 1; continue; }
if b == b',' { tokens.push(Token { kind: TokenKind::Comma, line: line_num, col }); i += 1; continue; }
if b == b'?' { tokens.push(Token { kind: TokenKind::Question, line: line_num, col }); i += 1; continue; }
if b == b':' { tokens.push(Token { kind: TokenKind::Colon, line: line_num, col }); i += 1; continue; }
if b == b'^' { tokens.push(Token { kind: TokenKind::Caret, line: line_num, col }); i += 1; continue; }
if b == b'=' {
if i + 1 < len && bytes[i + 1] == b'=' {
tokens.push(Token { kind: TokenKind::EqualEqual, line: line_num, col });
i += 2;
} else {
tokens.push(Token { kind: TokenKind::Equal, line: line_num, col });
i += 1;
}
continue;
}
if b == b'!' {
if i + 1 < len && bytes[i + 1] == b'=' {
tokens.push(Token { kind: TokenKind::NotEqual, line: line_num, col });
i += 2;
} else {
tokens.push(Token { kind: TokenKind::Not, line: line_num, col });
i += 1;
}
continue;
}
if b == b'<' {
if i + 1 < len && bytes[i + 1] == b'=' {
tokens.push(Token { kind: TokenKind::LessEqual, line: line_num, col });
i += 2;
} else {
tokens.push(Token { kind: TokenKind::Less, line: line_num, col });
i += 1;
}
continue;
}
if b == b'>' {
if i + 1 < len && bytes[i + 1] == b'=' {
tokens.push(Token { kind: TokenKind::GreaterEqual, line: line_num, col });
i += 2;
} else {
tokens.push(Token { kind: TokenKind::Greater, line: line_num, col });
i += 1;
}
continue;
}
if b == b'&' && i + 1 < len && bytes[i + 1] == b'&' {
tokens.push(Token { kind: TokenKind::And, line: line_num, col });
i += 2;
continue;
}
if b == b'|' && i + 1 < len && bytes[i + 1] == b'|' {
tokens.push(Token { kind: TokenKind::Or, line: line_num, col });
i += 2;
continue;
}
if b == b'+' { tokens.push(Token { kind: TokenKind::Plus, line: line_num, col }); i += 1; continue; }
if b == b'-' { tokens.push(Token { kind: TokenKind::Minus, line: line_num, col }); i += 1; continue; }
if b == b'*' { tokens.push(Token { kind: TokenKind::Star, line: line_num, col }); i += 1; continue; }
if b == b'/' { tokens.push(Token { kind: TokenKind::Slash, line: line_num, col }); i += 1; continue; }
if b == b'%' { tokens.push(Token { kind: TokenKind::Percent, line: line_num, col }); i += 1; continue; }
if b == b'"' {
i += 1;
let mut s = String::new();
let mut closed = false;
let remaining = &text[i..];
let mut chars = remaining.char_indices();
while let Some((c_idx, ch)) = chars.next() {
if ch == '\\' {
if let Some((_, next_ch)) = chars.next() {
match next_ch {
'n' => s.push('\n'),
't' => s.push('\t'),
'r' => s.push('\r'),
'"' => s.push('"'),
'\\' => s.push('\\'),
other => s.push(other),
}
} else {
break;
}
} else if ch == '"' {
closed = true;
i += c_idx + 1;
break;
} else {
s.push(ch);
}
}
if !closed {
return Err(PvgError::lex(line_num, col, "Unclosed string literal."));
}
tokens.push(Token { kind: TokenKind::String(s), line: line_num, col });
continue;
}
if b.is_ascii_digit() || (b == b'.' && i + 1 < len && bytes[i + 1].is_ascii_digit()) {
let start = i;
let mut has_dot = false;
while i < len && (bytes[i].is_ascii_digit() || (!has_dot && bytes[i] == b'.')) {
if bytes[i] == b'.' {
has_dot = true;
}
i += 1;
}
let num_str = std::str::from_utf8(&bytes[start..i]).unwrap();
let mut val = num_str
.parse::<f64>()
.map_err(|e| PvgError::lex(line_num, col, format!("Invalid number: {}", e)))?;
if i + 3 <= len && &bytes[i..i + 3] == b"deg" {
val = val.to_radians();
i += 3;
} else if i + 3 <= len && &bytes[i..i + 3] == b"rad" {
i += 3;
}
tokens.push(Token { kind: TokenKind::Number(val), line: line_num, col });
continue;
}
if b.is_ascii_alphabetic() || b == b'_' {
let start = i;
while i < len && (bytes[i].is_ascii_alphanumeric() || bytes[i] == b'_' || bytes[i] == b'-') {
i += 1;
}
let ident = std::str::from_utf8(&bytes[start..i]).unwrap();
let kind = match ident {
"PVG" | "CPSVG" => TokenKind::Pvg,
"canvas" => TokenKind::Canvas,
"background" => TokenKind::Background,
"snap" => TokenKind::Snap,
"filter" => TokenKind::Filter,
"param" => TokenKind::Param,
"pattern" => TokenKind::Pattern,
"sprite" => TokenKind::Sprite,
"spline" => TokenKind::Spline,
"palette" => TokenKind::Palette,
"data" => TokenKind::Data,
"row" => TokenKind::Row,
"set" => TokenKind::Set,
"def" => TokenKind::Def,
"return" => TokenKind::Return,
"for" => TokenKind::For,
"from" => TokenKind::From,
"to" => TokenKind::To,
"step" => TokenKind::Step,
"while" => TokenKind::While,
"if" => TokenKind::If,
"else" => TokenKind::Else,
"seed" => TokenKind::Seed,
"circle" => TokenKind::Circle,
"ellipse" => TokenKind::Ellipse,
"rectangle" | "rect" => TokenKind::Rectangle,
"line" => TokenKind::Line,
"polygon" => TokenKind::Polygon,
"path" => TokenKind::Path,
"text" => TokenKind::Text,
"group" => TokenKind::Group,
"clip" => TokenKind::Clip,
"center" => TokenKind::Center,
"radius" => TokenKind::Radius,
"pos" => TokenKind::Pos,
"size" => TokenKind::Size,
"points" => TokenKind::Points,
"content" => TokenKind::Content,
"font" => TokenKind::Font,
"align" => TokenKind::Align,
"fill" => TokenKind::Fill,
"stroke" => TokenKind::Stroke,
"width" => TokenKind::Width,
"opacity" => TokenKind::Opacity,
"rot" => TokenKind::Rot,
"scale" => TokenKind::Scale,
"cap" => TokenKind::Cap,
"join" => TokenKind::Join,
"miter" => TokenKind::Miter,
"dash" => TokenKind::Dash,
"blend" => TokenKind::Blend,
"blur" => TokenKind::Blur,
"shadow" => TokenKind::Shadow,
"glow" => TokenKind::Glow,
"linear" => TokenKind::Linear,
"radial" => TokenKind::Radial,
"angular" | "conic" => TokenKind::Angular,
"stop" => TokenKind::Stop,
"start" => TokenKind::Start,
"quad" => TokenKind::Quad,
"curve" => TokenKind::Curve,
"arc" => TokenKind::Arc,
"close" => TokenKind::Close,
"and" => TokenKind::And,
"or" => TokenKind::Or,
"not" => TokenKind::Not,
"black" => TokenKind::Color(Color::BLACK),
"white" => TokenKind::Color(Color::WHITE),
"red" => TokenKind::Color(Color::RED),
"green" => TokenKind::Color(Color::GREEN),
"blue" => TokenKind::Color(Color::BLUE),
"yellow" => TokenKind::Color(Color::YELLOW),
"cyan" => TokenKind::Color(Color::CYAN),
"magenta" => TokenKind::Color(Color::MAGENTA),
"none" | "transparent" => TokenKind::Color(Color::None),
"true" => TokenKind::Ident("true".into()),
"false" => TokenKind::Ident("false".into()),
_ => TokenKind::Ident(ident.to_string()),
};
tokens.push(Token { kind, line: line_num, col });
continue;
}
return Err(PvgError::lex(
line_num,
col,
format!("Unexpected character '{}'.", b as char),
));
}
Ok(())
}
}
fn triple_opener_before_comment(content: &str) -> bool {
let bytes = content.as_bytes();
let open = match content.find("\"\"\"") {
Some(i) => i,
None => return false,
};
let mut i = 0;
while i < open {
if bytes[i] == b'#' {
let mut hex_end = i + 1;
while hex_end < bytes.len() && bytes[hex_end].is_ascii_hexdigit() {
hex_end += 1;
}
let hex_len = hex_end - (i + 1);
if hex_len == 3 || hex_len == 6 || hex_len == 8 {
let is_delim = hex_end == bytes.len()
|| bytes[hex_end].is_ascii_whitespace()
|| matches!(bytes[hex_end], b']' | b')' | b',' | b':' | b'"');
if is_delim {
i = hex_end;
continue;
}
}
return false;
}
i += 1;
}
true
}