rpic_core/

lexer.rs

//! Hand-written lexer for the pic language.
//!
//! Recognises the full dpic token vocabulary (see `token.rs`). It is a simple
//! character scanner — pic sources are small, so we keep the whole input in a
//! `Vec<char>` and index into it. Line/column are tracked for diagnostics.

use std::collections::HashMap;
use std::sync::Arc;

use crate::diagnostic::Span;
use crate::token::*;

/// A token together with its source position (1-based line/column).
#[derive(Clone)]
pub struct Spanned {
    pub tok: Token,
    pub line: u32,
    pub col: u32,
    pub end_col: u32,
    /// Provenance: `None` = the user's own input, `Some` = the `copy` include
    /// or library the token was lexed from. Positions are relative to that
    /// source. Deliberately excluded from `PartialEq` — identical token
    /// streams compare equal regardless of where they were lexed.
    pub file: Option<Arc<str>>,
    /// Macro arguments that were in scope when this token was produced by
    /// macro substitution. Used by eval-time `exec` expansion.
    pub arg_frame: Option<Vec<Vec<Spanned>>>,
    /// Macro definitions that were in scope when a deferred `if`/`for` body was
    /// copied. Used when that body is parsed later by the evaluator.
    pub macro_frame: Option<Arc<HashMap<String, Vec<Spanned>>>>,
}

impl Spanned {
    pub fn new(tok: Token, line: u32, col: u32) -> Self {
        Self {
            tok,
            line,
            col,
            end_col: col.saturating_add(1),
            file: None,
            arg_frame: None,
            macro_frame: None,
        }
    }

    pub fn with_end(mut self, end_col: u32) -> Self {
        self.end_col = end_col;
        self
    }

    pub fn with_file(mut self, file: Option<Arc<str>>) -> Self {
        self.file = file;
        self
    }

    pub fn span(&self) -> Span {
        Span::new(self.line, self.col, self.end_col).in_file(self.file.clone())
    }

    pub fn with_arg_frame(mut self, args: &[Vec<Spanned>]) -> Self {
        self.arg_frame = Some(args.to_vec());
        self
    }

    pub fn with_macro_frame(mut self, macros: &HashMap<String, Vec<Spanned>>) -> Self {
        self.macro_frame = Some(Arc::new(macros.clone()));
        self
    }
}

impl std::fmt::Debug for Spanned {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        f.debug_struct("Spanned")
            .field("tok", &self.tok)
            .field("line", &self.line)
            .field("col", &self.col)
            .field("end_col", &self.end_col)
            .field("arg_frame", &self.arg_frame)
            .finish_non_exhaustive()
    }
}

impl PartialEq for Spanned {
    fn eq(&self, other: &Self) -> bool {
        self.tok == other.tok
            && self.line == other.line
            && self.col == other.col
            && self.end_col == other.end_col
            && self.arg_frame == other.arg_frame
    }
}

/// A lexing error with location.
#[derive(Debug, Clone, PartialEq)]
pub struct LexError {
    pub msg: String,
    pub line: u32,
    pub col: u32,
    pub end_col: u32,
    /// See [`Spanned::file`]: the include/library the error is in, if any.
    pub file: Option<Arc<str>>,
    pub kind: String,
}

impl std::fmt::Display for LexError {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        match &self.file {
            Some(file) => write!(f, "{}:{}:{}: {}", file, self.line, self.col, self.msg),
            None => write!(f, "{}:{}: {}", self.line, self.col, self.msg),
        }
    }
}

/// Tokenize `src`. The returned vector always ends with [`Token::Eof`].
pub fn lex(src: &str) -> Result<Vec<Spanned>, LexError> {
    Lexer::new(src, None).run()
}

/// Like [`lex`], tagging every token (and any error) with the name of the
/// source it came from — a `copy` include or a loaded library. Positions in
/// diagnostics then stay relative to that source instead of pointing into a
/// concatenated stream.
pub fn lex_named(src: &str, file: &str) -> Result<Vec<Spanned>, LexError> {
    Lexer::new(src, Some(Arc::from(file))).run()
}

struct Lexer {
    chars: Vec<char>,
    pos: usize,
    line: u32,
    col: u32,
    file: Option<Arc<str>>,
    out: Vec<Spanned>,
}

impl Lexer {
    fn new(src: &str, file: Option<Arc<str>>) -> Self {
        Lexer {
            chars: src.chars().collect(),
            pos: 0,
            line: 1,
            col: 1,
            file,
            out: Vec::new(),
        }
    }

    fn peek(&self) -> Option<char> {
        self.chars.get(self.pos).copied()
    }
    fn peek_at(&self, n: usize) -> Option<char> {
        self.chars.get(self.pos + n).copied()
    }

    /// Consume and return the next char, advancing line/column.
    fn bump(&mut self) -> Option<char> {
        let c = self.chars.get(self.pos).copied()?;
        self.pos += 1;
        if c == '\n' {
            self.line += 1;
            self.col = 1;
        } else {
            self.col += 1;
        }
        Some(c)
    }

    fn err<T>(&self, msg: impl Into<String>) -> Result<T, LexError> {
        self.err_at(self.line, self.col, self.col.saturating_add(1), "lex", msg)
    }

    fn err_at<T>(
        &self,
        line: u32,
        col: u32,
        end_col: u32,
        kind: impl Into<String>,
        msg: impl Into<String>,
    ) -> Result<T, LexError> {
        Err(LexError {
            msg: msg.into(),
            line,
            col,
            end_col,
            file: self.file.clone(),
            kind: kind.into(),
        })
    }

    fn run(mut self) -> Result<Vec<Spanned>, LexError> {
        loop {
            // Skip spaces, tabs, CR, and `#` comments. Backslash-newline is a
            // line continuation (both consumed).
            loop {
                match self.peek() {
                    Some(' ') | Some('\t') | Some('\r') => {
                        self.bump();
                    }
                    Some('#') => {
                        while let Some(c) = self.peek() {
                            if c == '\n' {
                                break;
                            }
                            self.bump();
                        }
                    }
                    Some('\\') => {
                        // continuation: `\` [spaces] newline
                        let save = (self.pos, self.line, self.col);
                        self.bump();
                        while matches!(self.peek(), Some(' ') | Some('\t') | Some('\r')) {
                            self.bump();
                        }
                        if self.peek() == Some('\n') {
                            self.bump();
                        } else {
                            // not a continuation; restore and let the main
                            // matcher report the stray backslash.
                            self.pos = save.0;
                            self.line = save.1;
                            self.col = save.2;
                            break;
                        }
                    }
                    _ => break,
                }
            }

            let (line, col) = (self.line, self.col);
            let c = match self.peek() {
                None => {
                    self.push(Token::Eof, line, col, col);
                    return Ok(self.out);
                }
                Some(c) => c,
            };

            let tok = if c == '\n' || c == ';' {
                self.bump();
                Token::Newline
            } else if c == '"' {
                self.lex_string()?
            } else if c == '$' {
                self.lex_arg()?
            } else if c.is_ascii_digit()
                || (c == '.' && self.peek_at(1).is_some_and(|d| d.is_ascii_digit()))
            {
                self.lex_number()?
            } else if c == '.' {
                self.lex_dot()?
            } else if c.is_alphabetic() || c == '_' {
                let tok = self.lex_word();
                if matches!(tok, Token::Kw(Kw::Sh | Kw::Command)) {
                    self.skip_raw_command_arg();
                }
                tok
            } else {
                match self.lex_operator()? {
                    Some(t) => t,
                    None => continue, // (should not happen)
                }
            };
            let end_col = if self.line == line {
                self.col
            } else {
                col.saturating_add(1)
            };
            self.push(tok, line, col, end_col);
        }
    }

    fn push(&mut self, tok: Token, line: u32, col: u32, end_col: u32) {
        // A newline adjacent to `then` is a line/spline-path continuation, not a
        // statement terminator: circuit_macros figures wrap long paths across
        // lines without a trailing `\`, breaking either after `then`
        // (`… then ⏎ left_ …`) or before it (`… 3*dimen_ ⏎ then …`). `then` only
        // occurs inside a path and always needs a following element, so dropping
        // the newline is unambiguous in both directions.
        if tok == Token::Newline
            && matches!(self.out.last().map(|s| &s.tok), Some(Token::Kw(Kw::Then)))
        {
            return;
        }
        if tok == Token::Kw(Kw::Then)
            && matches!(self.out.last().map(|s| &s.tok), Some(Token::Newline))
        {
            self.out.pop();
        }
        self.out.push(
            Spanned::new(tok, line, col)
                .with_end(end_col)
                .with_file(self.file.clone()),
        );
    }

    fn lex_string(&mut self) -> Result<Token, LexError> {
        let (start_line, start_col) = (self.line, self.col);
        self.bump(); // opening quote
        let mut s = String::new();
        loop {
            match self.bump() {
                None => {
                    return self.err_at(
                        start_line,
                        start_col,
                        start_col.saturating_add(1),
                        "unterminated_string",
                        "unterminated string literal",
                    );
                }
                Some('"') => break,
                Some('\\') => {
                    // Preserve escape sequences (e.g. troff escapes) verbatim;
                    // a backslash-quote does not terminate the string.
                    s.push('\\');
                    match self.bump() {
                        Some(c) => s.push(c),
                        None => {
                            return self.err_at(
                                start_line,
                                start_col,
                                start_col.saturating_add(1),
                                "unterminated_string",
                                "unterminated string literal",
                            );
                        }
                    }
                }
                Some(c) => s.push(c),
            }
        }
        Ok(Token::Str(s))
    }

    fn skip_raw_command_arg(&mut self) {
        while matches!(self.peek(), Some(' ') | Some('\t') | Some('\r')) {
            self.bump();
        }
        if self.peek() == Some('"') {
            self.skip_raw_quoted();
        } else if self.starts_word_here("sprintf") {
            for _ in 0.."sprintf".len() {
                self.bump();
            }
            while matches!(self.peek(), Some(' ') | Some('\t') | Some('\r')) {
                self.bump();
            }
            if self.peek() == Some('(') {
                self.skip_raw_parens();
            }
        } else {
            self.skip_raw_line_tail();
        }
    }

    fn starts_word_here(&self, word: &str) -> bool {
        for (i, want) in word.chars().enumerate() {
            if self.peek_at(i) != Some(want) {
                return false;
            }
        }
        !self
            .peek_at(word.len())
            .is_some_and(|c| c.is_alphanumeric() || c == '_')
    }

    fn skip_raw_quoted(&mut self) {
        if self.peek() != Some('"') {
            return;
        }
        self.bump();
        while let Some(c) = self.bump() {
            match c {
                '\\' => {
                    self.bump();
                }
                '"' if self.raw_quote_is_followed_by(&['\n', ';', '}']) => break,
                _ => {}
            }
        }
    }

    fn skip_raw_parens(&mut self) {
        let mut depth = 0i32;
        while let Some(c) = self.bump() {
            match c {
                '"' => self.skip_raw_quoted_tail(),
                '(' => depth += 1,
                ')' => {
                    depth -= 1;
                    if depth == 0 {
                        break;
                    }
                }
                _ => {}
            }
        }
    }

    fn skip_raw_quoted_tail(&mut self) {
        while let Some(c) = self.bump() {
            match c {
                '\\' => {
                    self.bump();
                }
                '"' if self.raw_quote_is_followed_by(&['\n', ';', '}', ',', ')']) => break,
                _ => {}
            }
        }
    }

    fn raw_quote_is_followed_by(&self, stops: &[char]) -> bool {
        let mut off = 0;
        loop {
            match self.peek_at(off) {
                Some(' ' | '\t' | '\r') => off += 1,
                Some(c) => return stops.contains(&c),
                None => return true,
            }
        }
    }

    fn skip_raw_line_tail(&mut self) {
        loop {
            let mut last_non_ws = None;
            while let Some(c) = self.peek() {
                if c == '\n' {
                    break;
                }
                if !matches!(c, ' ' | '\t' | '\r') {
                    last_non_ws = Some(c);
                }
                self.bump();
            }
            if self.peek() == Some('\n') && last_non_ws == Some('\\') {
                self.bump();
                continue;
            }
            break;
        }
    }

    fn lex_arg(&mut self) -> Result<Token, LexError> {
        self.bump(); // '$'
        // `$+` is the count of arguments to the current macro
        if self.peek() == Some('+') {
            self.bump();
            return Ok(Token::ArgCount);
        }
        let mut n = String::new();
        while let Some(c) = self.peek() {
            if c.is_ascii_digit() {
                n.push(c);
                self.bump();
            } else {
                break;
            }
        }
        if n.is_empty() {
            // A `$` not followed by a digit or `+` is a literal `$` (e.g. `$f$`
            // LaTeX text passed unquoted as a macro argument), as in m4/dpic.
            return Ok(Token::Dollar);
        }
        match n.parse() {
            Ok(v) => Ok(Token::Arg(v)),
            Err(_) => self.err(format!("macro argument `${n}` is too large")),
        }
    }

    fn lex_number(&mut self) -> Result<Token, LexError> {
        // rpic extension (pikchr-compatible): `0x…` hexadecimal literals,
        // mainly for colour values (`shaded 0x1b5e20`). `0x` without a hex
        // digit stays classic: Num(0) followed by a name.
        if self.peek() == Some('0')
            && matches!(self.peek_at(1), Some('x') | Some('X'))
            && self.peek_at(2).is_some_and(|d| d.is_ascii_hexdigit())
        {
            self.bump();
            self.bump();
            let mut v: f64 = 0.0;
            while let Some(c) = self.peek() {
                if let Some(d) = c.to_digit(16) {
                    v = v * 16.0 + d as f64;
                    self.bump();
                } else {
                    break;
                }
            }
            return Ok(Token::Float(v));
        }
        let mut s = String::new();
        if self.peek() == Some('.') {
            s.push('0'); // ".5" -> "0.5"
        }
        while let Some(c) = self.peek() {
            if c.is_ascii_digit() {
                s.push(c);
                self.bump();
            } else {
                break;
            }
        }
        // fractional part: only consume `.` when a digit follows, so a trailing
        // `.` (e.g. in `box.ne`) stays a separate token.
        if self.peek() == Some('.') && self.peek_at(1).is_some_and(|d| d.is_ascii_digit()) {
            s.push('.');
            self.bump();
            while let Some(c) = self.peek() {
                if c.is_ascii_digit() {
                    s.push(c);
                    self.bump();
                } else {
                    break;
                }
            }
        }
        // exponent
        if matches!(self.peek(), Some('e') | Some('E'))
            && (self.peek_at(1).is_some_and(|d| d.is_ascii_digit())
                || (matches!(self.peek_at(1), Some('+') | Some('-'))
                    && self.peek_at(2).is_some_and(|d| d.is_ascii_digit())))
        {
            s.push('e');
            self.bump();
            if matches!(self.peek(), Some('+') | Some('-')) {
                s.push(self.bump().unwrap());
            }
            while let Some(c) = self.peek() {
                if c.is_ascii_digit() {
                    s.push(c);
                    self.bump();
                } else {
                    break;
                }
            }
        }
        // optional inch unit suffix `i`/`I` (pic's base unit is the inch, so the
        // suffix is informational); don't consume it when it begins an
        // identifier, e.g. the `in` of a following word.
        if matches!(self.peek(), Some('i') | Some('I'))
            && !self
                .peek_at(1)
                .is_some_and(|c| c.is_alphanumeric() || c == '_')
        {
            self.bump();
        }
        match s.parse::<f64>() {
            Ok(v) if v.is_finite() => Ok(Token::Float(v)),
            Ok(_) => self.err(format!("number `{s}` is not finite")),
            Err(_) => self.err(format!("invalid number `{s}`")),
        }
    }

    /// Lex a `.`-prefixed token: `.PS/.PE`, compass corners, `.x/.y`, dotted
    /// attribute accessors, or a bare `.` (the placename separator).
    fn lex_dot(&mut self) -> Result<Token, LexError> {
        let save = (self.pos, self.line, self.col);
        self.bump(); // '.'
        // read the following word
        let mut w = String::new();
        while let Some(c) = self.peek() {
            if c.is_alphanumeric() || c == '_' {
                w.push(c);
                self.bump();
            } else {
                break;
            }
        }
        if let Some(tok) = dot_keyword(&w) {
            Ok(tok)
        } else {
            // Not a dotted keyword: emit a bare `.` and rewind so the word is
            // lexed normally on the next pass.
            self.pos = save.0 + 1;
            self.line = save.1;
            self.col = save.2 + 1;
            Ok(Token::Dot)
        }
    }

    fn lex_word(&mut self) -> Token {
        let mut w = String::new();
        while let Some(c) = self.peek() {
            if c.is_alphanumeric() || c == '_' {
                w.push(c);
                self.bump();
            } else {
                break;
            }
        }
        word_keyword(&w)
    }

    fn lex_operator(&mut self) -> Result<Option<Token>, LexError> {
        let c = self.bump().unwrap();
        let next = self.peek();
        let tok = match c {
            '(' => Token::Lparen,
            ')' => Token::Rparen,
            ',' => Token::Comma,
            '^' => Token::Caret,
            '{' => Token::LeftBrace,
            '}' => Token::RightBrace,
            ']' => Token::RightBrack,
            '`' => Token::LeftQuote,
            '\'' => Token::RightQuote,
            '[' => {
                if next == Some(']') {
                    self.bump();
                    Token::Block
                } else {
                    Token::LeftBrack
                }
            }
            ':' => self.two('=', Token::ColonEq, Token::Colon),
            '=' => self.two('=', Token::EqEq, Token::Eq),
            '!' => self.two('=', Token::Neq, Token::Not),
            '+' => self.two('=', Token::PlusEq, Token::Plus),
            '*' => self.two('=', Token::MultEq, Token::Mult),
            '/' => self.two('=', Token::DivEq, Token::Div),
            '%' => self.two('=', Token::RemEq, Token::Percent),
            '>' => self.two('=', Token::Ge, Token::Gt),
            '&' => self.two('&', Token::AndAnd, Token::Ampersand),
            '|' => {
                if next == Some('|') {
                    self.bump();
                    Token::OrOr
                } else {
                    return self.err("unexpected `|`");
                }
            }
            '-' => match next {
                Some('>') => {
                    self.bump();
                    Token::Arrow(Arrow::Right)
                }
                Some('=') => {
                    self.bump();
                    Token::MinusEq
                }
                _ => Token::Minus,
            },
            '<' => match next {
                Some('=') => {
                    self.bump();
                    Token::Le
                }
                Some('-') => {
                    self.bump();
                    if self.peek() == Some('>') {
                        self.bump();
                        Token::Arrow(Arrow::Double)
                    } else {
                        Token::Arrow(Arrow::Left)
                    }
                }
                _ => Token::Lt,
            },
            // A `\` that is not a line continuation (handled in the whitespace
            // skip) is literal text — e.g. a LaTeX command like `\beta` passed
            // unquoted as a macro argument, as in m4/dpic. (`c` is already
            // consumed by the `bump()` above.)
            '\\' => Token::Backslash,
            other => return self.err(format!("unexpected character `{other}`")),
        };
        Ok(Some(tok))
    }

    /// If the next char is `c`, consume it and return `yes`; otherwise `no`.
    fn two(&mut self, c: char, yes: Token, no: Token) -> Token {
        if self.peek() == Some(c) {
            self.bump();
            yes
        } else {
            no
        }
    }
}

/// Classify a bare word (after the keyword vocabulary). Unknown words become a
/// [`Token::Label`] (upper-initial) or [`Token::Name`] (otherwise).
fn word_keyword(w: &str) -> Token {
    use Token::*;
    match w {
        // primitives
        "box" => Prim(self::Prim::Box),
        "circle" => Prim(self::Prim::Circle),
        "ellipse" => Prim(self::Prim::Ellipse),
        "arc" => Prim(self::Prim::Arc),
        "line" => Prim(self::Prim::Line),
        "arrow" => Prim(self::Prim::Arrow),
        "move" => Prim(self::Prim::Move),
        "spline" => Prim(self::Prim::Spline),
        // directions
        "up" => Dir(self::Dir::Up),
        "down" => Dir(self::Dir::Down),
        "right" => Dir(self::Dir::Right),
        "left" => Dir(self::Dir::Left),
        // attributes & joiners
        "height" | "ht" => Kw(self::Kw::Ht),
        "width" | "wid" => Kw(self::Kw::Wid),
        "radius" | "rad" => Kw(self::Kw::Rad),
        "diameter" | "diam" => Kw(self::Kw::Diam),
        "thickness" | "thick" => Kw(self::Kw::Thick),
        "scaled" => Kw(self::Kw::Scaled),
        "from" => Kw(self::Kw::From),
        "to" => Kw(self::Kw::To),
        "at" => Kw(self::Kw::At),
        "with" => Kw(self::Kw::With),
        "by" => Kw(self::Kw::By),
        "then" => Kw(self::Kw::Then),
        "cw" => Kw(self::Kw::Cw),
        "ccw" => Kw(self::Kw::Ccw),
        "continue" => Kw(self::Kw::Continue),
        "chop" => Kw(self::Kw::Chop),
        "same" => Kw(self::Kw::Same),
        "of" => Kw(self::Kw::Of),
        "the" => Kw(self::Kw::The),
        "way" => Kw(self::Kw::Way),
        "between" => Kw(self::Kw::Between),
        "and" => Kw(self::Kw::And),
        "last" => Kw(self::Kw::Last),
        "fill" | "filled" => Kw(self::Kw::Fill),
        "st" | "nd" | "rd" | "th" => Kw(self::Kw::Nth),
        "Here" => Kw(self::Kw::Here),
        // bare corner words
        "top" => Corner(self::Corner::N),
        "bottom" => Corner(self::Corner::S),
        "start" => Corner(self::Corner::Start),
        "end" => Corner(self::Corner::End),
        // commands / control
        "print" => Kw(self::Kw::Print),
        "copy" => Kw(self::Kw::Copy),
        "reset" => Kw(self::Kw::Reset),
        "exec" => Kw(self::Kw::Exec),
        "sh" => Kw(self::Kw::Sh),
        "command" => Kw(self::Kw::Command),
        "define" => Kw(self::Kw::Define),
        "undefine" | "undef" => Kw(self::Kw::Undef),
        "rand" => Kw(self::Kw::Rand),
        "if" => Kw(self::Kw::If),
        "else" => Kw(self::Kw::Else),
        "for" => Kw(self::Kw::For),
        "do" => Kw(self::Kw::Do),
        "sprintf" => Kw(self::Kw::Sprintf),
        // rpic animation extension
        "animate" => Kw(self::Kw::Animate),
        "after" => Kw(self::Kw::After),
        "delay" => Kw(self::Kw::Delay),
        // line types
        "solid" => LineType(self::LineType::Solid),
        "dotted" => LineType(self::LineType::Dotted),
        "dashed" => LineType(self::LineType::Dashed),
        "invis" | "invisible" => LineType(self::LineType::Invis),
        // color / outline / shade
        "color" | "colour" | "colored" | "coloured" => Color(self::Color::Colored),
        "outline" | "outlined" => Color(self::Color::Outlined),
        "shade" | "shaded" => Color(self::Color::Shaded),
        // text position
        "center" | "centre" => TextPos(self::TextPos::Center),
        "ljust" => TextPos(self::TextPos::Ljust),
        "rjust" => TextPos(self::TextPos::Rjust),
        "above" => TextPos(self::TextPos::Above),
        "below" => TextPos(self::TextPos::Below),
        // one-arg functions
        "abs" => Func1(self::Func1::Abs),
        "acos" => Func1(self::Func1::Acos),
        "asin" => Func1(self::Func1::Asin),
        "cos" => Func1(self::Func1::Cos),
        "exp" => Func1(self::Func1::Exp),
        "expe" => Func1(self::Func1::Expe),
        "int" => Func1(self::Func1::Int),
        "log" => Func1(self::Func1::Log),
        "loge" => Func1(self::Func1::Loge),
        "sign" => Func1(self::Func1::Sign),
        "sin" => Func1(self::Func1::Sin),
        "sqrt" => Func1(self::Func1::Sqrt),
        "tan" => Func1(self::Func1::Tan),
        "floor" => Func1(self::Func1::Floor),
        // two-arg functions
        "atan2" => Func2(self::Func2::Atan2),
        "max" => Func2(self::Func2::Max),
        "min" => Func2(self::Func2::Min),
        "pmod" => Func2(self::Func2::Pmod),
        // environment variables
        "arcrad" => EnvVar(self::EnvVar::Arcrad),
        "arrowht" => EnvVar(self::EnvVar::Arrowht),
        "arrowwid" => EnvVar(self::EnvVar::Arrowwid),
        "boxht" => EnvVar(self::EnvVar::Boxht),
        "boxrad" => EnvVar(self::EnvVar::Boxrad),
        "boxwid" => EnvVar(self::EnvVar::Boxwid),
        "circlerad" => EnvVar(self::EnvVar::Circlerad),
        "dashwid" => EnvVar(self::EnvVar::Dashwid),
        "ellipseht" => EnvVar(self::EnvVar::Ellipseht),
        "ellipsewid" => EnvVar(self::EnvVar::Ellipsewid),
        "lineht" => EnvVar(self::EnvVar::Lineht),
        "linewid" => EnvVar(self::EnvVar::Linewid),
        "moveht" => EnvVar(self::EnvVar::Moveht),
        "movewid" => EnvVar(self::EnvVar::Movewid),
        "textht" => EnvVar(self::EnvVar::Textht),
        "textoffset" => EnvVar(self::EnvVar::Textoffset),
        "textwid" => EnvVar(self::EnvVar::Textwid),
        "arrowhead" => EnvVar(self::EnvVar::Arrowhead),
        "fillval" => EnvVar(self::EnvVar::Fillval),
        "linethick" => EnvVar(self::EnvVar::Linethick),
        "margin" => EnvVar(self::EnvVar::Margin),
        "topmargin" => EnvVar(self::EnvVar::Topmargin),
        "rightmargin" => EnvVar(self::EnvVar::Rightmargin),
        "bottommargin" => EnvVar(self::EnvVar::Bottommargin),
        "leftmargin" => EnvVar(self::EnvVar::Leftmargin),
        "maxpsht" => EnvVar(self::EnvVar::Maxpsht),
        "maxpswid" => EnvVar(self::EnvVar::Maxpswid),
        "scale" => EnvVar(self::EnvVar::Scale),
        "texlabels" => EnvVar(self::EnvVar::Texlabels),
        "dotrad" => EnvVar(self::EnvVar::Dotrad),
        // identifier / label
        _ => {
            if w.chars().next().is_some_and(|c| c.is_ascii_uppercase()) {
                Label(w.to_string())
            } else {
                Name(w.to_string())
            }
        }
    }
}

/// Classify the word following a `.`. Returns `None` if it is not a dotted
/// keyword (so the caller emits a bare `.`).
fn dot_keyword(w: &str) -> Option<Token> {
    use Token::*;
    let t = match w {
        "PS" => DotPS,
        "PE" => DotPE,
        "x" => DotX,
        "y" => DotY,
        // compass corners
        "ne" => Corner(self::Corner::Ne),
        "se" => Corner(self::Corner::Se),
        "nw" => Corner(self::Corner::Nw),
        "sw" => Corner(self::Corner::Sw),
        "n" | "t" | "top" | "north" => Corner(self::Corner::N),
        "s" | "b" | "bot" | "bottom" | "south" => Corner(self::Corner::S),
        "e" | "r" | "east" | "right" => Corner(self::Corner::E),
        "w" | "l" | "west" | "left" => Corner(self::Corner::W),
        "start" => Corner(self::Corner::Start),
        "end" => Corner(self::Corner::End),
        "c" | "center" | "centre" => Corner(self::Corner::Center),
        // dotted attribute accessors
        "ht" | "height" => Param(self::Param::Height),
        "wid" | "width" => Param(self::Param::Width),
        "rad" | "radius" => Param(self::Param::Radius),
        "diam" | "diameter" => Param(self::Param::Diameter),
        "thick" | "thickness" => Param(self::Param::Thickness),
        "len" | "length" => Param(self::Param::Length),
        _ => return None,
    };
    Some(t)
}

#[cfg(test)]
mod tests {
    use super::*;

    fn toks(src: &str) -> Vec<Token> {
        lex(src).unwrap().into_iter().map(|s| s.tok).collect()
    }

    #[test]
    fn basic_box() {
        assert_eq!(
            toks("box \"hi\""),
            vec![Token::Prim(Prim::Box), Token::Str("hi".into()), Token::Eof]
        );
    }

    #[test]
    fn label_vs_name() {
        assert_eq!(
            toks("Start: boxwid"),
            vec![
                Token::Label("Start".into()),
                Token::Colon,
                Token::EnvVar(EnvVar::Boxwid),
                Token::Eof
            ]
        );
        // a lower-initial non-keyword is a Name
        assert_eq!(toks("myvar"), vec![Token::Name("myvar".into()), Token::Eof]);
    }

    #[test]
    fn numbers() {
        assert_eq!(toks("0.5"), vec![Token::Float(0.5), Token::Eof]);
        assert_eq!(toks(".25"), vec![Token::Float(0.25), Token::Eof]);
        assert_eq!(toks("1e3"), vec![Token::Float(1000.0), Token::Eof]);
        assert_eq!(toks("2.5e-1"), vec![Token::Float(0.25), Token::Eof]);
    }

    #[test]
    fn rejects_non_finite_numbers() {
        let err = lex("box wid 1e999").unwrap_err();
        assert!(err.msg.contains("not finite"), "{err}");
    }

    #[test]
    fn operators_and_arrows() {
        assert_eq!(
            toks("a := b <= c -> d <- e <-> f"),
            vec![
                Token::Name("a".into()),
                Token::ColonEq,
                Token::Name("b".into()),
                Token::Le,
                Token::Name("c".into()),
                Token::Arrow(Arrow::Right),
                Token::Name("d".into()),
                Token::Arrow(Arrow::Left),
                Token::Name("e".into()),
                Token::Arrow(Arrow::Double),
                Token::Name("f".into()),
                Token::Eof
            ]
        );
    }

    #[test]
    fn minus_is_not_arrow_without_gt() {
        assert_eq!(
            toks("2-3"),
            vec![
                Token::Float(2.0),
                Token::Minus,
                Token::Float(3.0),
                Token::Eof
            ]
        );
    }

    #[test]
    fn compass_and_params() {
        assert_eq!(
            toks("last box.ne A.ht .center"),
            vec![
                Token::Kw(Kw::Last),
                Token::Prim(Prim::Box),
                Token::Corner(Corner::Ne),
                Token::Label("A".into()),
                Token::Param(Param::Height),
                Token::Corner(Corner::Center),
                Token::Eof
            ]
        );
    }

    #[test]
    fn dot_then_label() {
        // `.` not followed by a dotted keyword is a bare Dot, then the word.
        assert_eq!(
            toks("B.A"),
            vec![
                Token::Label("B".into()),
                Token::Dot,
                Token::Label("A".into()),
                Token::Eof
            ]
        );
    }

    #[test]
    fn statement_separators_and_comments() {
        assert_eq!(
            toks("box; circle # a comment\narc"),
            vec![
                Token::Prim(Prim::Box),
                Token::Newline,
                Token::Prim(Prim::Circle),
                Token::Newline,
                Token::Prim(Prim::Arc),
                Token::Eof
            ]
        );
    }

    #[test]
    fn shell_commands_skip_raw_line_tail() {
        assert_eq!(
            toks("sh \"echo -n \\\"print \\\\\\\"\\\" > $1_prow\"\nbox"),
            vec![
                Token::Kw(Kw::Sh),
                Token::Newline,
                Token::Prim(Prim::Box),
                Token::Eof
            ]
        );
        assert_eq!(
            toks("command \\foo $bad \"unterminated\ncircle"),
            vec![
                Token::Kw(Kw::Command),
                Token::Newline,
                Token::Prim(Prim::Circle),
                Token::Eof
            ]
        );
        assert_eq!(
            toks("sh \"sed something \\\n  > tmp\"\narc"),
            vec![
                Token::Kw(Kw::Sh),
                Token::Newline,
                Token::Prim(Prim::Arc),
                Token::Eof
            ]
        );
        assert_eq!(
            toks("sh \"rm\";}\ncommand sprintf(\"x\", y) }\n"),
            vec![
                Token::Kw(Kw::Sh),
                Token::Newline,
                Token::RightBrace,
                Token::Newline,
                Token::Kw(Kw::Command),
                Token::RightBrace,
                Token::Newline,
                Token::Eof
            ]
        );
        assert_eq!(
            toks("sh \"echo -n \\\"print \\\\\"\\\" > $1_prow\"\nbox"),
            vec![
                Token::Kw(Kw::Sh),
                Token::Newline,
                Token::Prim(Prim::Box),
                Token::Eof
            ]
        );
    }

    #[test]
    fn line_continuation() {
        assert_eq!(
            toks("box \\\n  wid 2"),
            vec![
                Token::Prim(Prim::Box),
                Token::Kw(Kw::Wid),
                Token::Float(2.0),
                Token::Eof
            ]
        );
    }

    #[test]
    fn ps_pe_and_block() {
        assert_eq!(
            toks(".PS\nbox\n.PE"),
            vec![
                Token::DotPS,
                Token::Newline,
                Token::Prim(Prim::Box),
                Token::Newline,
                Token::DotPE,
                Token::Eof
            ]
        );
        assert_eq!(toks("[]"), vec![Token::Block, Token::Eof]);
        assert_eq!(
            toks("[ box ]"),
            vec![
                Token::LeftBrack,
                Token::Prim(Prim::Box),
                Token::RightBrack,
                Token::Eof
            ]
        );
    }

    #[test]
    fn macro_arg() {
        assert_eq!(
            toks("box $1"),
            vec![Token::Prim(Prim::Box), Token::Arg(1), Token::Eof]
        );
    }

    #[test]
    fn func_and_envvar() {
        assert_eq!(
            toks("sqrt(2) atan2 scale margin topmargin"),
            vec![
                Token::Func1(Func1::Sqrt),
                Token::Lparen,
                Token::Float(2.0),
                Token::Rparen,
                Token::Func2(Func2::Atan2),
                Token::EnvVar(EnvVar::Scale),
                Token::EnvVar(EnvVar::Margin),
                Token::EnvVar(EnvVar::Topmargin),
                Token::Eof
            ]
        );
    }

    #[test]
    fn newline_after_then_is_continuation() {
        // `then` ⏎ continues a path (no statement break); a normal newline does not.
        assert_eq!(
            toks("line right then\nup"),
            vec![
                Token::Prim(Prim::Line),
                Token::Dir(Dir::Right),
                Token::Kw(Kw::Then),
                Token::Dir(Dir::Up),
                Token::Eof,
            ]
        );
        assert_eq!(
            toks("box\nup"),
            vec![
                Token::Prim(Prim::Box),
                Token::Newline,
                Token::Dir(Dir::Up),
                Token::Eof,
            ]
        );
    }

    #[test]
    fn newline_before_then_is_continuation() {
        // a path may also wrap *before* `then` (`… right\nthen up`).
        assert_eq!(
            toks("line right\nthen up"),
            vec![
                Token::Prim(Prim::Line),
                Token::Dir(Dir::Right),
                Token::Kw(Kw::Then),
                Token::Dir(Dir::Up),
                Token::Eof,
            ]
        );
    }

    #[test]
    fn dollar_and_backslash_are_literal_text() {
        // `$` not before a digit/`+`, and a non-continuation `\`, are literal
        // text (e.g. `$\beta$` LaTeX passed unquoted as a macro argument).
        assert_eq!(
            toks("$\\beta$"),
            vec![
                Token::Dollar,
                Token::Backslash,
                Token::Name("beta".into()),
                Token::Dollar,
                Token::Eof,
            ]
        );
        // `$1` is still a macro argument.
        assert_eq!(toks("$1"), vec![Token::Arg(1), Token::Eof]);
    }
}