latex_rust/parser/
token.rs

1//! LaTeX math tokenizer.
2
3use core::fmt;
4use core::iter::Peekable;
5use core::str::Chars;
6
7use crate::error::ParseError;
8
9/// A single TeX-style math token.
10#[derive(Clone, Debug, PartialEq, Eq)]
11pub enum Token {
12    /// Ordinary character (letter, digit, or other).
13    Char(char),
14    /// Control sequence without the leading backslash (`frac`, `[`, `,`).
15    Command(String),
16    /// `{`
17    BeginGroup,
18    /// `}`
19    EndGroup,
20    /// `^`
21    Superscript,
22    /// `_`
23    Subscript,
24    /// `&`
25    AlignmentTab,
26    /// Single `$`
27    MathShift,
28    /// `$$`
29    DisplayShift,
30    /// A space character (kept for `\text`; skipped in math lists).
31    Space,
32}
33
34impl fmt::Display for Token {
35    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
36        match self {
37            Self::Char(c) => write!(f, "char:{c}"),
38            Self::Command(name) => write!(f, "cmd:{name}"),
39            Self::BeginGroup => f.write_str("{"),
40            Self::EndGroup => f.write_str("}"),
41            Self::Superscript => f.write_str("^"),
42            Self::Subscript => f.write_str("_"),
43            Self::AlignmentTab => f.write_str("&"),
44            Self::MathShift => f.write_str("$"),
45            Self::DisplayShift => f.write_str("$$"),
46            Self::Space => f.write_str("space"),
47        }
48    }
49}
50
51/// Format a token stream as a gold-stable string.
52///
53/// # Examples
54///
55/// ```
56/// use latex_rust::{format_tokens, tokenize};
57///
58/// let t = tokenize(r"a^2").unwrap();
59/// assert_eq!(format_tokens(&t), "char:a ^ char:2");
60/// ```
61#[must_use]
62pub fn format_tokens(tokens: &[Token]) -> String {
63    let mut out = String::new();
64    for (i, t) in tokens.iter().enumerate() {
65        if i > 0 {
66            out.push(' ');
67        }
68        out.push_str(&t.to_string());
69    }
70    out
71}
72
73/// Tokenize a LaTeX math string. Whitespace and `%` line comments are skipped.
74///
75/// Supported delimiters are tokenized, not interpreted:
76/// `$...$`, `$$...$$`, `\[...\]`, `\(...\)`.
77///
78/// # Arguments
79///
80/// * `input` — math source.
81///
82/// # Returns
83///
84/// The token list in source order.
85///
86/// # Errors
87///
88/// [`ParseError::TrailingBackslash`] if `input` ends with a stray `\`.
89///
90/// # Examples
91///
92/// ```
93/// use latex_rust::{tokenize, Token};
94///
95/// let t = tokenize(r"\frac{1}{2}").unwrap();
96/// assert!(matches!(&t[0], Token::Command(s) if s == "frac"));
97/// ```
98pub fn tokenize(input: &str) -> Result<Vec<Token>, ParseError> {
99    let mut chars = input.chars().peekable();
100    let mut out = Vec::new();
101    while let Some(c) = chars.next() {
102        match c {
103            '%' => skip_line(&mut chars),
104            ' ' => out.push(Token::Space),
105            '\t' | '\n' | '\r' => {}
106            '{' => out.push(Token::BeginGroup),
107            '}' => out.push(Token::EndGroup),
108            '^' => out.push(Token::Superscript),
109            '_' => out.push(Token::Subscript),
110            '&' => out.push(Token::AlignmentTab),
111            '$' => {
112                if chars.peek() == Some(&'$') {
113                    chars.next();
114                    out.push(Token::DisplayShift);
115                } else {
116                    out.push(Token::MathShift);
117                }
118            }
119            '\\' => out.push(command(&mut chars)?),
120            other => out.push(Token::Char(other)),
121        }
122    }
123    Ok(out)
124}
125
126fn skip_line(chars: &mut Peekable<Chars<'_>>) {
127    for c in chars.by_ref() {
128        if c == '\n' {
129            break;
130        }
131    }
132}
133
134fn command(chars: &mut Peekable<Chars<'_>>) -> Result<Token, ParseError> {
135    let Some(&first) = chars.peek() else {
136        return Err(ParseError::TrailingBackslash);
137    };
138    if first.is_ascii_alphabetic() {
139        let mut name = String::new();
140        while let Some(&c) = chars.peek() {
141            if c.is_ascii_alphabetic() {
142                name.push(c);
143                chars.next();
144            } else {
145                break;
146            }
147        }
148        while matches!(chars.peek(), Some(' ' | '\t' | '\n' | '\r')) {
149            chars.next();
150        }
151        Ok(Token::Command(name))
152    } else {
153        chars.next();
154        Ok(Token::Command(first.to_string()))
155    }
156}
latex_rust/parser/token.rs

latex_rust/parser/
token.rs
