ratex_parser/
parser.rs

1use ratex_lexer::token::{SourceLocation, Token};
2use unicode_normalization::UnicodeNormalization;
3
4use crate::error::{ParseError, ParseResult};
5use crate::functions::{self, ArgType, FunctionContext, FUNCTIONS};
6use crate::macro_expander::{MacroExpander, IMPLICIT_COMMANDS};
7use crate::parse_node::{AtomFamily, Mode, ParseNode};
8
9/// End-of-expression tokens.
10static END_OF_EXPRESSION: &[&str] = &["}", "\\endgroup", "\\end", "\\right", "&"];
11
12const MAX_RECURSION_DEPTH: usize = 512;
13
14/// The LaTeX parser. Converts a token stream into a ParseNode AST.
15///
16/// Follows KaTeX's Parser.ts closely:
17/// - `parse()` → parse full expression
18/// - `parseExpression()` → parse a list of atoms
19/// - `parseAtom()` → parse one atom with optional super/subscripts
20/// - `parseGroup()` → parse a group (braced or single token)
21/// - `parseFunction()` → parse a function call with arguments
22/// - `parseSymbol()` → parse a single symbol
23pub struct Parser<'a> {
24    pub mode: Mode,
25    pub gullet: MacroExpander<'a>,
26    pub leftright_depth: i32,
27    recursion_depth: usize,
28    next_token: Option<Token>,
29    pub equation_counter: usize,
30}
31
32impl<'a> Parser<'a> {
33    pub fn new(input: &'a str) -> Self {
34        Self {
35            mode: Mode::Math,
36            gullet: MacroExpander::new(input, Mode::Math),
37            leftright_depth: 0,
38            recursion_depth: 0,
39            next_token: None,
40            equation_counter: 0,
41        }
42    }
43
44    // ── Token management ────────────────────────────────────────────────
45
46    /// Return the current lookahead token (fetching from gullet if needed).
47    pub fn fetch(&mut self) -> ParseResult<Token> {
48        if self.next_token.is_none() {
49            self.next_token = Some(self.gullet.expand_next_token()?);
50        }
51        Ok(self.next_token.clone().unwrap())
52    }
53
54    /// Discard the current lookahead token.
55    pub fn consume(&mut self) {
56        self.next_token = None;
57    }
58
59    /// Expect the next token to have the given text, consuming it.
60    pub fn expect(&mut self, text: &str, do_consume: bool) -> ParseResult<()> {
61        let tok = self.fetch()?;
62        if tok.text != text {
63            return Err(ParseError::new(
64                format!("Expected '{}', got '{}'", text, tok.text),
65                Some(&tok),
66            ));
67        }
68        if do_consume {
69            self.consume();
70        }
71        Ok(())
72    }
73
74    /// Consume spaces in math mode.
75    pub fn consume_spaces(&mut self) -> ParseResult<()> {
76        loop {
77            let tok = self.fetch()?;
78            if tok.text == " " {
79                self.consume();
80            } else {
81                break;
82            }
83        }
84        Ok(())
85    }
86
87    /// Switch between "math" and "text" modes.
88    pub fn switch_mode(&mut self, new_mode: Mode) {
89        self.mode = new_mode;
90        self.gullet.switch_mode(new_mode);
91    }
92
93    // ── Main parse entry ────────────────────────────────────────────────
94
95    /// Parse the entire input and return the AST.
96    pub fn parse(&mut self) -> ParseResult<Vec<ParseNode>> {
97        self.gullet.begin_group();
98
99        let result = self.parse_expression(false, None);
100
101        match result {
102            Ok(parse) => {
103                self.expect("EOF", true)?;
104                self.gullet.end_group();
105                Ok(parse)
106            }
107            Err(e) => {
108                self.gullet.end_groups();
109                Err(e)
110            }
111        }
112    }
113
114    // ── Expression parsing ──────────────────────────────────────────────
115
116    /// Parse an expression: a list of atoms.
117    pub fn parse_expression(
118        &mut self,
119        break_on_infix: bool,
120        break_on_token_text: Option<&str>,
121    ) -> ParseResult<Vec<ParseNode>> {
122        self.recursion_depth += 1;
123        if self.recursion_depth > MAX_RECURSION_DEPTH {
124            self.recursion_depth -= 1;
125            return Err(ParseError::recursion_limit_exceeded());
126        }
127        let result = self.parse_expression_impl(break_on_infix, break_on_token_text);
128        self.recursion_depth -= 1;
129        result
130    }
131
132    fn parse_expression_impl(
133        &mut self,
134        break_on_infix: bool,
135        break_on_token_text: Option<&str>,
136    ) -> ParseResult<Vec<ParseNode>> {
137        let mut body = Vec::new();
138
139        loop {
140            if self.mode == Mode::Math {
141                self.consume_spaces()?;
142            }
143
144            let lex = self.fetch()?;
145
146            if END_OF_EXPRESSION.contains(&lex.text.as_str()) {
147                break;
148            }
149            if let Some(break_text) = break_on_token_text {
150                if lex.text == break_text {
151                    break;
152                }
153            }
154            if break_on_infix {
155                if let Some(func) = FUNCTIONS.get(lex.text.as_str()) {
156                    if func.infix {
157                        break;
158                    }
159                }
160            }
161
162            let atom = self.parse_atom(break_on_token_text)?;
163
164            match atom {
165                None => break,
166                Some(node) if node.type_name() == "internal" => continue,
167                Some(node) => body.push(node),
168            }
169        }
170
171        if self.mode == Mode::Text {
172            self.form_ligatures(&mut body);
173        }
174
175        self.handle_infix_nodes(body)
176    }
177
178    /// Rewrite infix operators (e.g. \over → \frac).
179    fn handle_infix_nodes(&mut self, body: Vec<ParseNode>) -> ParseResult<Vec<ParseNode>> {
180        let mut over_index: Option<usize> = None;
181        let mut func_name: Option<String> = None;
182
183        for (i, node) in body.iter().enumerate() {
184            if let ParseNode::Infix { replace_with, .. } = node {
185                if over_index.is_some() {
186                    return Err(ParseError::msg("only one infix operator per group"));
187                }
188                over_index = Some(i);
189                func_name = Some(replace_with.clone());
190            }
191        }
192
193        if let (Some(idx), Some(fname)) = (over_index, func_name) {
194            let numer_body: Vec<ParseNode> = body[..idx].to_vec();
195            let denom_body: Vec<ParseNode> = body[idx + 1..].to_vec();
196
197            let numer = if numer_body.len() == 1 {
198                if let ParseNode::OrdGroup { .. } = &numer_body[0] {
199                    numer_body.into_iter().next().unwrap()
200                } else {
201                    ParseNode::OrdGroup {
202                        mode: self.mode,
203                        body: numer_body,
204                        semisimple: None,
205                        loc: None,
206                    }
207                }
208            } else {
209                ParseNode::OrdGroup {
210                    mode: self.mode,
211                    body: numer_body,
212                    semisimple: None,
213                    loc: None,
214                }
215            };
216
217            let denom = if denom_body.len() == 1 {
218                if let ParseNode::OrdGroup { .. } = &denom_body[0] {
219                    denom_body.into_iter().next().unwrap()
220                } else {
221                    ParseNode::OrdGroup {
222                        mode: self.mode,
223                        body: denom_body,
224                        semisimple: None,
225                        loc: None,
226                    }
227                }
228            } else {
229                ParseNode::OrdGroup {
230                    mode: self.mode,
231                    body: denom_body,
232                    semisimple: None,
233                    loc: None,
234                }
235            };
236
237            let node = if fname == "\\\\abovefrac" {
238                // \above passes the infix node (with bar size) as the middle argument
239                let infix_node = body[idx].clone();
240                self.call_function(&fname, vec![numer, infix_node, denom], vec![], None, None)?
241            } else {
242                self.call_function(&fname, vec![numer, denom], vec![], None, None)?
243            };
244            Ok(vec![node])
245        } else {
246            Ok(body)
247        }
248    }
249
250    /// Form ligatures in text mode (e.g. -- → –, --- → —).
251    fn form_ligatures(&self, group: &mut Vec<ParseNode>) {
252        let mut i = 0;
253        while i + 1 < group.len() {
254            let a_text = group[i].symbol_text().map(|s| s.to_string());
255            let b_text = group[i + 1].symbol_text().map(|s| s.to_string());
256
257            if let (Some(a), Some(b)) = (a_text, b_text) {
258                if group[i].type_name() == "textord" && group[i + 1].type_name() == "textord" {
259                    if a == "-" && b == "-" {
260                        if i + 2 < group.len() {
261                            if let Some(c) = group[i + 2].symbol_text() {
262                                if c == "-" && group[i + 2].type_name() == "textord" {
263                                    group[i] = ParseNode::TextOrd {
264                                        mode: Mode::Text,
265                                        text: "---".to_string(),
266                                        loc: None,
267                                    };
268                                    group.remove(i + 2);
269                                    group.remove(i + 1);
270                                    continue;
271                                }
272                            }
273                        }
274                        group[i] = ParseNode::TextOrd {
275                            mode: Mode::Text,
276                            text: "--".to_string(),
277                            loc: None,
278                        };
279                        group.remove(i + 1);
280                        continue;
281                    }
282                    if (a == "'" || a == "`") && b == a {
283                        group[i] = ParseNode::TextOrd {
284                            mode: Mode::Text,
285                            text: format!("{}{}", a, a),
286                            loc: None,
287                        };
288                        group.remove(i + 1);
289                        continue;
290                    }
291                }
292            }
293            i += 1;
294        }
295    }
296
297    // ── Atom parsing ────────────────────────────────────────────────────
298
299    /// Parse a single atom with optional super/subscripts.
300    pub fn parse_atom(
301        &mut self,
302        break_on_token_text: Option<&str>,
303    ) -> ParseResult<Option<ParseNode>> {
304        let mut base = self.parse_group("atom", break_on_token_text)?;
305
306        if let Some(ref b) = base {
307            if b.type_name() == "internal" {
308                return Ok(base);
309            }
310        }
311
312        if self.mode == Mode::Text {
313            return Ok(base);
314        }
315
316        let mut superscript: Option<ParseNode> = None;
317        let mut subscript: Option<ParseNode> = None;
318
319        loop {
320            self.consume_spaces()?;
321            let lex = self.fetch()?;
322
323            if lex.text == "\\limits" || lex.text == "\\nolimits" {
324                let is_limits = lex.text == "\\limits";
325                self.consume();
326                if let Some(base_node) = base.as_mut() {
327                    match base_node {
328                        ParseNode::Op {
329                            limits,
330                            always_handle_sup_sub,
331                            ..
332                        } => {
333                            *limits = is_limits;
334                            *always_handle_sup_sub = Some(is_limits);
335                        }
336                        ParseNode::OperatorName {
337                            limits,
338                            always_handle_sup_sub,
339                            ..
340                        } => {
341                            *limits = is_limits;
342                            *always_handle_sup_sub = is_limits;
343                        }
344                        _ => {}
345                    }
346                }
347            } else if lex.text == "^" {
348                if superscript.is_some() {
349                    return Err(ParseError::new("Double superscript", Some(&lex)));
350                }
351                superscript = Some(self.handle_sup_subscript("superscript")?);
352            } else if lex.text == "_" {
353                if subscript.is_some() {
354                    return Err(ParseError::new("Double subscript", Some(&lex)));
355                }
356                subscript = Some(self.handle_sup_subscript("subscript")?);
357            } else if lex.text == "'" {
358                if superscript.is_some() {
359                    return Err(ParseError::new("Double superscript", Some(&lex)));
360                }
361                let prime = ParseNode::TextOrd {
362                    mode: self.mode,
363                    text: "\\prime".to_string(),
364                    loc: None,
365                };
366                let mut primes = vec![prime.clone()];
367                self.consume();
368                while self.fetch()?.text == "'" {
369                    primes.push(prime.clone());
370                    self.consume();
371                }
372                if self.fetch()?.text == "^" {
373                    primes.push(self.handle_sup_subscript("superscript")?);
374                }
375                superscript = Some(ParseNode::OrdGroup {
376                    mode: self.mode,
377                    body: primes,
378                    semisimple: None,
379                    loc: None,
380                });
381            } else if let Some((mapped, is_sub)) = lex
382                .text
383                .chars()
384                .next()
385                .and_then(crate::unicode_sup_sub::unicode_sub_sup)
386            {
387                if is_sub && subscript.is_some() {
388                    return Err(ParseError::new("Double subscript", Some(&lex)));
389                }
390                if !is_sub && superscript.is_some() {
391                    return Err(ParseError::new("Double superscript", Some(&lex)));
392                }
393                // Collect consecutive Unicode sup/sub chars of the same kind
394                let mut subsup_tokens = vec![Token::new(mapped, 0, 0)];
395                self.consume();
396                loop {
397                    let tok = self.fetch()?;
398                    match tok
399                        .text
400                        .chars()
401                        .next()
402                        .and_then(crate::unicode_sup_sub::unicode_sub_sup)
403                    {
404                        Some((m, sub)) if sub == is_sub => {
405                            subsup_tokens.insert(0, Token::new(m, 0, 0));
406                            self.consume();
407                        }
408                        _ => break,
409                    }
410                }
411                let body = self.subparse(subsup_tokens)?;
412                let group = ParseNode::OrdGroup {
413                    mode: Mode::Math,
414                    body,
415                    semisimple: None,
416                    loc: None,
417                };
418                if is_sub {
419                    subscript = Some(group);
420                } else {
421                    superscript = Some(group);
422                }
423            } else {
424                break;
425            }
426        }
427
428        if superscript.is_some() || subscript.is_some() {
429            Ok(Some(ParseNode::SupSub {
430                mode: self.mode,
431                base: base.map(Box::new),
432                sup: superscript.map(Box::new),
433                sub: subscript.map(Box::new),
434                loc: None,
435            }))
436        } else {
437            Ok(base)
438        }
439    }
440
441    /// Handle a subscript or superscript.
442    fn handle_sup_subscript(&mut self, name: &str) -> ParseResult<ParseNode> {
443        let symbol_token = self.fetch()?;
444        self.consume();
445        self.consume_spaces()?;
446
447        let group = self.parse_group(name, None)?;
448        match group {
449            Some(g) if g.type_name() != "internal" => Ok(g),
450            Some(_) => {
451                // Skip internal nodes, try again
452                let g2 = self.parse_group(name, None)?;
453                g2.ok_or_else(|| {
454                    ParseError::new(
455                        format!("Expected group after '{}'", symbol_token.text),
456                        Some(&symbol_token),
457                    )
458                })
459            }
460            None => Err(ParseError::new(
461                format!("Expected group after '{}'", symbol_token.text),
462                Some(&symbol_token),
463            )),
464        }
465    }
466
467    // ── Group parsing ───────────────────────────────────────────────────
468
469    /// Parse a group: braced expression, function call, or single symbol.
470    pub fn parse_group(
471        &mut self,
472        name: &str,
473        break_on_token_text: Option<&str>,
474    ) -> ParseResult<Option<ParseNode>> {
475        let first_token = self.fetch()?;
476        let text = first_token.text.clone();
477
478        if text == "{" || text == "\\begingroup" {
479            self.consume();
480            let group_end = if text == "{" { "}" } else { "\\endgroup" };
481
482            self.gullet.begin_group();
483            let expression = self.parse_expression(false, Some(group_end))?;
484            let last_token = self.fetch()?;
485            self.expect(group_end, true)?;
486            self.gullet.end_group();
487
488            let loc = Some(SourceLocation::range(&first_token.loc, &last_token.loc));
489            let semisimple = if text == "\\begingroup" {
490                Some(true)
491            } else {
492                None
493            };
494
495            Ok(Some(ParseNode::OrdGroup {
496                mode: self.mode,
497                body: expression,
498                semisimple,
499                loc,
500            }))
501        } else {
502            let result = self
503                .parse_function(break_on_token_text, Some(name))?
504                .or_else(|| self.parse_symbol_inner().ok().flatten());
505
506            if result.is_none()
507                && text.starts_with('\\')
508                && !IMPLICIT_COMMANDS.contains(&text.as_str())
509            {
510                return Err(ParseError::new(
511                    format!("Undefined control sequence: {}", text),
512                    Some(&first_token),
513                ));
514            }
515
516            Ok(result)
517        }
518    }
519
520    // ── Function parsing ────────────────────────────────────────────────
521
522    /// Try to parse a function call. Returns None if not a function.
523    pub fn parse_function(
524        &mut self,
525        break_on_token_text: Option<&str>,
526        name: Option<&str>,
527    ) -> ParseResult<Option<ParseNode>> {
528        let token = self.fetch()?;
529        let func = token.text.clone();
530
531        let func_data = match FUNCTIONS.get(func.as_str()) {
532            Some(f) => f,
533            None => return Ok(None),
534        };
535
536        self.consume();
537
538        if let Some(n) = name {
539            if n != "atom" && !func_data.allowed_in_argument {
540                return Err(ParseError::new(
541                    format!("Got function '{}' with no arguments as {}", func, n),
542                    Some(&token),
543                ));
544            }
545        }
546
547        functions::check_mode_compatibility(func_data, self.mode, &func, Some(&token))?;
548
549        // `\hspace*{len}` — `*` is a separate token (not part of the control word); consume it here.
550        // Must use gullet peek/pop only: `parser.fetch()` without `consume()` advances the lexer and
551        // leaves `{` only in `next_token`, so `parse_size_group`'s `gullet.future()` would miss the brace.
552        if func == "\\hspace" {
553            self.gullet.consume_spaces();
554            if self.gullet.future().text == "*" {
555                self.gullet.pop_token();
556            }
557        }
558
559        let (args, opt_args) = self.parse_arguments(&func, func_data)?;
560
561        self.call_function(
562            &func,
563            args,
564            opt_args,
565            Some(token),
566            break_on_token_text.map(|s| s.to_string()).as_deref(),
567        )
568        .map(Some)
569    }
570
571    /// Call a function handler.
572    pub fn call_function(
573        &mut self,
574        name: &str,
575        args: Vec<ParseNode>,
576        opt_args: Vec<Option<ParseNode>>,
577        token: Option<Token>,
578        break_on_token_text: Option<&str>,
579    ) -> ParseResult<ParseNode> {
580        let func = FUNCTIONS.get(name).ok_or_else(|| {
581            ParseError::msg(format!("No function handler for {}", name))
582        })?;
583
584        let mut ctx = FunctionContext {
585            func_name: name.to_string(),
586            parser: self,
587            token: token.clone(),
588            break_on_token_text: break_on_token_text.map(|s| s.to_string()),
589        };
590
591        (func.handler)(&mut ctx, args, opt_args)
592    }
593
594    /// Parse the arguments for a function.
595    pub fn parse_arguments(
596        &mut self,
597        func: &str,
598        func_data: &functions::FunctionSpec,
599    ) -> ParseResult<(Vec<ParseNode>, Vec<Option<ParseNode>>)> {
600        let total_args = func_data.num_args + func_data.num_optional_args;
601        if total_args == 0 {
602            return Ok((Vec::new(), Vec::new()));
603        }
604
605        let mut args = Vec::new();
606        let mut opt_args = Vec::new();
607
608        for i in 0..total_args {
609            let arg_type = func_data
610                .arg_types
611                .as_ref()
612                .and_then(|types| types.get(i).copied());
613            let is_optional = i < func_data.num_optional_args;
614
615            let effective_type = if (func_data.primitive && arg_type.is_none())
616                || (func_data.node_type == "sqrt" && i == 1
617                    && opt_args.first().is_some_and(|o: &Option<ParseNode>| o.is_none()))
618            {
619                Some(ArgType::Primitive)
620            } else {
621                arg_type
622            };
623
624            let arg = self.parse_group_of_type(
625                &format!("argument to '{}'", func),
626                effective_type,
627                is_optional,
628            )?;
629
630            if is_optional {
631                opt_args.push(arg);
632            } else if let Some(a) = arg {
633                args.push(a);
634            } else {
635                return Err(ParseError::msg("Null argument, please report this as a bug"));
636            }
637        }
638
639        Ok((args, opt_args))
640    }
641
642    /// Parse a group with a specific type.
643    fn parse_group_of_type(
644        &mut self,
645        name: &str,
646        arg_type: Option<ArgType>,
647        optional: bool,
648    ) -> ParseResult<Option<ParseNode>> {
649        match arg_type {
650            Some(ArgType::Color) => self.parse_color_group(optional),
651            Some(ArgType::Size) => self.parse_size_group(optional),
652            Some(ArgType::Primitive) => {
653                if optional {
654                    return Err(ParseError::msg("A primitive argument cannot be optional"));
655                }
656                let group = self.parse_group(name, None)?;
657                match group {
658                    Some(g) => Ok(Some(g)),
659                    None => Err(ParseError::new(
660                        format!("Expected group as {}", name),
661                        None,
662                    )),
663                }
664            }
665            Some(ArgType::Math) | Some(ArgType::Text) => {
666                let mode = match arg_type {
667                    Some(ArgType::Math) => Some(Mode::Math),
668                    Some(ArgType::Text) => Some(Mode::Text),
669                    _ => None,
670                };
671                self.parse_argument_group(optional, mode)
672            }
673            Some(ArgType::HBox) => {
674                let group = self.parse_argument_group(optional, Some(Mode::Text))?;
675                match group {
676                    Some(g) => Ok(Some(ParseNode::Styling {
677                        mode: g.mode(),
678                        style: crate::parse_node::StyleStr::Text,
679                        body: vec![g],
680                        loc: None,
681                    })),
682                    None => Ok(None),
683                }
684            }
685            Some(ArgType::Raw) => {
686                let token = self.parse_string_group("raw", optional)?;
687                match token {
688                    Some(t) => Ok(Some(ParseNode::Raw {
689                        mode: Mode::Text,
690                        string: t.text,
691                        loc: None,
692                    })),
693                    None => Ok(None),
694                }
695            }
696            Some(ArgType::Url) => self.parse_url_group(optional),
697            None | Some(ArgType::Original) => self.parse_argument_group(optional, None),
698        }
699    }
700
701    /// Parse a color group.
702    fn parse_color_group(&mut self, optional: bool) -> ParseResult<Option<ParseNode>> {
703        let res = self.parse_string_group("color", optional)?;
704        match res {
705            None => Ok(None),
706            Some(token) => {
707                let text = token.text.trim().to_string();
708                let re = regex_lite::Regex::new(
709                    r"^(#[a-fA-F0-9]{3,4}|#[a-fA-F0-9]{6}|#[a-fA-F0-9]{8}|[a-fA-F0-9]{6}|[a-zA-Z]+|\d+(\.\d+)?(,\d+(\.\d+)?)*)$",
710                )
711                .unwrap();
712
713                if !re.is_match(&text) {
714                    return Err(ParseError::new(
715                        format!("Invalid color: '{}'", text),
716                        Some(&token),
717                    ));
718                }
719                let mut color = text;
720                if regex_lite::Regex::new(r"^[0-9a-fA-F]{6}$")
721                    .unwrap()
722                    .is_match(&color)
723                {
724                    color = format!("#{}", color);
725                }
726
727                Ok(Some(ParseNode::ColorToken {
728                    mode: self.mode,
729                    color,
730                    loc: None,
731                }))
732            }
733        }
734    }
735
736    /// Parse a size group (e.g., "3pt", "1em").
737    pub fn parse_size_group(&mut self, optional: bool) -> ParseResult<Option<ParseNode>> {
738        let mut is_blank = false;
739
740        self.gullet.consume_spaces();
741        let res = if !optional && self.gullet.future().text != "{" {
742            Some(self.parse_regex_group(
743                &regex_lite::Regex::new(r"^[-+]? *(?:$|\d+|\d+\.\d*|\.\d*) *[a-z]{0,2} *$")
744                    .unwrap(),
745                "size",
746            )?)
747        } else {
748            self.parse_string_group("size", optional)?
749        };
750
751        let res = match res {
752            Some(r) => r,
753            None => return Ok(None),
754        };
755
756        let mut text = res.text.clone();
757        if !optional && text.is_empty() {
758            text = "0pt".to_string();
759            is_blank = true;
760        }
761
762        let size_re =
763            regex_lite::Regex::new(r"([-+]?) *(\d+(?:\.\d*)?|\.\d+) *([a-z]{2})").unwrap();
764        let m = size_re.captures(&text).ok_or_else(|| {
765            ParseError::new(format!("Invalid size: '{}'", text), Some(&res))
766        })?;
767
768        let sign = m.get(1).map_or("", |m| m.as_str());
769        let magnitude = m.get(2).map_or("", |m| m.as_str());
770        let unit = m.get(3).map_or("", |m| m.as_str());
771
772        let number: f64 = format!("{}{}", sign, magnitude).parse().unwrap_or(0.0);
773
774        if !is_valid_unit(unit) {
775            return Err(ParseError::new(
776                format!("Invalid unit: '{}'", unit),
777                Some(&res),
778            ));
779        }
780
781        Ok(Some(ParseNode::Size {
782            mode: self.mode,
783            value: crate::parse_node::Measurement {
784                number,
785                unit: unit.to_string(),
786            },
787            is_blank,
788            loc: None,
789        }))
790    }
791
792    /// Parse a URL group.
793    /// Temporarily disables `%` as comment character to allow `%20` etc. in URLs.
794    fn parse_url_group(&mut self, optional: bool) -> ParseResult<Option<ParseNode>> {
795        self.gullet.lexer.set_catcode('%', 13);
796        self.gullet.lexer.set_catcode('~', 12);
797        let res = self.parse_string_group("url", optional);
798        self.gullet.lexer.set_catcode('%', 14);
799        self.gullet.lexer.set_catcode('~', 13);
800        let res = res?;
801        match res {
802            None => Ok(None),
803            Some(token) => {
804                let url = token.text;
805                Ok(Some(ParseNode::Url {
806                    mode: self.mode,
807                    url,
808                    loc: None,
809                }))
810            }
811        }
812    }
813
814    /// Parse a string group (brace-enclosed string).
815    fn parse_string_group(
816        &mut self,
817        _mode_name: &str,
818        optional: bool,
819    ) -> ParseResult<Option<Token>> {
820        let arg_token = self.gullet.scan_argument(optional)?;
821        let arg_token = match arg_token {
822            Some(t) => t,
823            None => return Ok(None),
824        };
825
826        let mut s = String::new();
827        loop {
828            let next = self.fetch()?;
829            if next.text == "EOF" {
830                break;
831            }
832            s.push_str(&next.text);
833            self.consume();
834        }
835        self.consume(); // consume EOF
836
837        let mut result = arg_token;
838        result.text = s;
839        Ok(Some(result))
840    }
841
842    /// Parse a regex-delimited group.
843    fn parse_regex_group(
844        &mut self,
845        regex: &regex_lite::Regex,
846        mode_name: &str,
847    ) -> ParseResult<Token> {
848        let first_token = self.fetch()?;
849        let mut last_token = first_token.clone();
850        let mut s = String::new();
851
852        loop {
853            let next = self.fetch()?;
854            if next.text == "EOF" {
855                break;
856            }
857            let candidate = format!("{}{}", s, next.text);
858            if regex.is_match(&candidate) {
859                last_token = next;
860                s = candidate;
861                self.consume();
862            } else {
863                break;
864            }
865        }
866
867        if s.is_empty() {
868            return Err(ParseError::new(
869                format!("Invalid {}: '{}'", mode_name, first_token.text),
870                Some(&first_token),
871            ));
872        }
873
874        Ok(first_token.range(&last_token, s))
875    }
876
877    /// Parse an argument group (with optional mode switch).
878    pub fn parse_argument_group(
879        &mut self,
880        optional: bool,
881        mode: Option<Mode>,
882    ) -> ParseResult<Option<ParseNode>> {
883        let arg_token = self.gullet.scan_argument(optional)?;
884        let arg_token = match arg_token {
885            Some(t) => t,
886            None => return Ok(None),
887        };
888
889        let outer_mode = self.mode;
890        if let Some(m) = mode {
891            self.switch_mode(m);
892        }
893
894        self.gullet.begin_group();
895        let expression = self.parse_expression(false, Some("EOF"))?;
896        self.expect("EOF", true)?;
897        self.gullet.end_group();
898
899        let result = ParseNode::OrdGroup {
900            mode: self.mode,
901            loc: Some(arg_token.loc.clone()),
902            body: expression,
903            semisimple: None,
904        };
905
906        if mode.is_some() {
907            self.switch_mode(outer_mode);
908        }
909
910        Ok(Some(result))
911    }
912
913    // ── Symbol parsing ──────────────────────────────────────────────────
914
915    /// Parse a single symbol (internal version that returns Result).
916    fn parse_symbol_inner(&mut self) -> ParseResult<Option<ParseNode>> {
917        let nucleus = self.fetch()?;
918        let text = nucleus.text.clone();
919
920        if let Some(stripped) = text.strip_prefix("\\verb") {
921            self.consume();
922            let (star, rest) = if let Some(r) = stripped.strip_prefix('*') {
923                (true, r)
924            } else {
925                (false, stripped)
926            };
927            let chars: Vec<char> = rest.chars().collect();
928            if chars.len() < 2 {
929                return Err(ParseError::new("\\verb assertion failed", Some(&nucleus)));
930            }
931            let body: String = chars[1..chars.len() - 1].iter().collect();
932            return Ok(Some(ParseNode::Verb {
933                mode: Mode::Text,
934                body,
935                star,
936                loc: Some(nucleus.loc.clone()),
937            }));
938        }
939
940        let font_mode = match self.mode {
941            Mode::Math => ratex_font::symbols::Mode::Math,
942            Mode::Text => ratex_font::symbols::Mode::Text,
943        };
944
945        // ^ and _ are handled by parse_atom for sup/sub, not as symbol nodes
946        if text == "^" || text == "_" {
947            return Ok(None);
948        }
949
950        // Bare backslash (incomplete control sequence) → not a valid symbol
951        if text == "\\" {
952            return Ok(None);
953        }
954
955        if let Some(sym_info) = ratex_font::symbols::get_symbol(&text, font_mode) {
956            let loc = Some(SourceLocation::range(&nucleus.loc, &nucleus.loc));
957            let group = sym_info.group;
958
959            let node = if group.is_atom() {
960                let family = match group {
961                    ratex_font::symbols::Group::Bin => AtomFamily::Bin,
962                    ratex_font::symbols::Group::Close => AtomFamily::Close,
963                    ratex_font::symbols::Group::Inner => AtomFamily::Inner,
964                    ratex_font::symbols::Group::Open => AtomFamily::Open,
965                    ratex_font::symbols::Group::Punct => AtomFamily::Punct,
966                    ratex_font::symbols::Group::Rel => AtomFamily::Rel,
967                    _ => unreachable!(),
968                };
969                ParseNode::Atom {
970                    mode: self.mode,
971                    family,
972                    text: text.clone(),
973                    loc,
974                }
975            } else {
976                match group {
977                    ratex_font::symbols::Group::MathOrd => ParseNode::MathOrd {
978                        mode: self.mode,
979                        text: text.clone(),
980                        loc,
981                    },
982                    ratex_font::symbols::Group::TextOrd => ParseNode::TextOrd {
983                        mode: self.mode,
984                        text: text.clone(),
985                        loc,
986                    },
987                    ratex_font::symbols::Group::OpToken => ParseNode::OpToken {
988                        mode: self.mode,
989                        text: text.clone(),
990                        loc,
991                    },
992                    ratex_font::symbols::Group::AccentToken => ParseNode::AccentToken {
993                        mode: self.mode,
994                        text: text.clone(),
995                        loc,
996                    },
997                    ratex_font::symbols::Group::Spacing => ParseNode::SpacingNode {
998                        mode: self.mode,
999                        text: text.clone(),
1000                        loc,
1001                    },
1002                    _ => ParseNode::MathOrd {
1003                        mode: self.mode,
1004                        text: text.clone(),
1005                        loc,
1006                    },
1007                }
1008            };
1009
1010            self.consume();
1011            return Ok(Some(node));
1012        }
1013
1014        // Unicode accented characters → decompose into accent nodes
1015        // Handles both precomposed (á U+00E1) and combining forms (a + U+0301)
1016        if let Some(node) = self.try_parse_unicode_accent(&text, &nucleus)? {
1017            self.consume();
1018            return Ok(Some(node));
1019        }
1020
1021        // Non-ASCII characters without accent decomposition → treat as textord
1022        // KaTeX always uses mode="text" for these, regardless of current mode
1023        let first_char = text.chars().next();
1024        if let Some(ch) = first_char {
1025            if ch as u32 >= 0x80 {
1026                let node = ParseNode::TextOrd {
1027                    mode: Mode::Text,
1028                    text: text.clone(),
1029                    loc: Some(SourceLocation::range(&nucleus.loc, &nucleus.loc)),
1030                };
1031                self.consume();
1032                return Ok(Some(node));
1033            }
1034        }
1035
1036        Ok(None)
1037    }
1038
1039    /// Try to decompose a Unicode accented character into accent nodes.
1040    /// Returns None if no decomposition is available.
1041    /// Only decomposes Latin-script characters, matching KaTeX behavior.
1042    fn try_parse_unicode_accent(
1043        &self,
1044        text: &str,
1045        nucleus: &Token,
1046    ) -> ParseResult<Option<ParseNode>> {
1047        let nfd: String = text.nfd().collect();
1048        let chars: Vec<char> = nfd.chars().collect();
1049
1050        if chars.len() < 2 {
1051            return Ok(None);
1052        }
1053
1054        // Build from the base up through each combining mark
1055        let mut split_idx = chars.len() - 1;
1056        while split_idx > 0 && is_supported_combining_accent(chars[split_idx]) {
1057            split_idx -= 1;
1058        }
1059
1060        // Verify ALL trailing chars are supported combining accents
1061        if split_idx == chars.len() - 1 {
1062            return Ok(None);
1063        }
1064
1065        // Only decompose Latin-script base characters
1066        let base_char = chars[0];
1067        if !is_latin_base_char(base_char) {
1068            return Ok(None);
1069        }
1070
1071        let loc = Some(SourceLocation::range(&nucleus.loc, &nucleus.loc));
1072
1073        // Base: everything before the combining marks
1074        let mut base_str: String = chars[..split_idx + 1].iter().collect();
1075
1076        // Accented i→ı and j→ȷ (dotless variants), matching KaTeX behavior
1077        if base_str.len() == 1 {
1078            match base_str.as_str() {
1079                "i" => base_str = "\u{0131}".to_string(), // ı
1080                "j" => base_str = "\u{0237}".to_string(), // ȷ
1081                _ => {}
1082            }
1083        }
1084
1085        let font_mode = match self.mode {
1086            Mode::Math => ratex_font::symbols::Mode::Math,
1087            Mode::Text => ratex_font::symbols::Mode::Text,
1088        };
1089
1090        let mut node = if base_str.chars().count() == 1 {
1091            let ch = base_str.chars().next().unwrap();
1092            if let Some(sym) = ratex_font::symbols::get_symbol(&base_str, font_mode) {
1093                match sym.group {
1094                    ratex_font::symbols::Group::TextOrd => ParseNode::TextOrd {
1095                        mode: self.mode,
1096                        text: base_str.clone(),
1097                        loc: loc.clone(),
1098                    },
1099                    _ => ParseNode::MathOrd {
1100                        mode: self.mode,
1101                        text: base_str.clone(),
1102                        loc: loc.clone(),
1103                    },
1104                }
1105            } else if (ch as u32) >= 0x80 {
1106                // Non-ASCII base chars always text mode (KaTeX compat)
1107                ParseNode::TextOrd {
1108                    mode: Mode::Text,
1109                    text: base_str.clone(),
1110                    loc: loc.clone(),
1111                }
1112            } else {
1113                ParseNode::MathOrd {
1114                    mode: self.mode,
1115                    text: base_str.clone(),
1116                    loc: loc.clone(),
1117                }
1118            }
1119        } else {
1120            return self.try_parse_unicode_accent(&base_str, nucleus).map(|opt| {
1121                opt.or_else(|| {
1122                    Some(ParseNode::TextOrd {
1123                        mode: Mode::Text,
1124                        text: base_str.clone(),
1125                        loc: loc.clone(),
1126                    })
1127                })
1128            });
1129        };
1130
1131        // Wrap in accent nodes from innermost to outermost
1132        for &combining in &chars[split_idx + 1..] {
1133            let label = combining_to_accent_label(combining, self.mode);
1134            node = ParseNode::Accent {
1135                mode: self.mode,
1136                label,
1137                is_stretchy: Some(false),
1138                is_shifty: Some(true),
1139                base: Box::new(node),
1140                loc: loc.clone(),
1141            };
1142        }
1143
1144        Ok(Some(node))
1145    }
1146
1147    /// Parse a sub-expression from the given tokens.
1148    pub fn subparse(&mut self, tokens: Vec<Token>) -> ParseResult<Vec<ParseNode>> {
1149        let old_token = self.next_token.take();
1150
1151        self.gullet
1152            .push_token(Token::new("}", 0, 0));
1153        self.gullet.push_tokens(tokens);
1154        let parse = self.parse_expression(false, None)?;
1155        self.expect("}", true)?;
1156
1157        self.next_token = old_token;
1158        Ok(parse)
1159    }
1160}
1161
1162fn is_latin_base_char(ch: char) -> bool {
1163    matches!(ch,
1164        'A'..='Z' | 'a'..='z'
1165        | '\u{0131}' // ı (dotless i)
1166        | '\u{0237}' // ȷ (dotless j)
1167        | '\u{00C6}' // Æ
1168        | '\u{00D0}' // Ð
1169        | '\u{00D8}' // Ø
1170        | '\u{00DE}' // Þ
1171        | '\u{00DF}' // ß
1172        | '\u{00E6}' // æ
1173        | '\u{00F0}' // ð
1174        | '\u{00F8}' // ø
1175        | '\u{00FE}' // þ
1176    )
1177}
1178
1179fn is_supported_combining_accent(ch: char) -> bool {
1180    matches!(
1181        ch,
1182        '\u{0300}' | '\u{0301}' | '\u{0302}' | '\u{0303}' | '\u{0304}'
1183        | '\u{0306}' | '\u{0307}' | '\u{0308}' | '\u{030A}' | '\u{030B}' | '\u{030C}'
1184        | '\u{0327}'
1185    )
1186}
1187
1188fn combining_to_accent_label(ch: char, mode: Mode) -> String {
1189    match mode {
1190        Mode::Math => match ch {
1191            '\u{0300}' => "\\grave".to_string(),
1192            '\u{0301}' => "\\acute".to_string(),
1193            '\u{0302}' => "\\hat".to_string(),
1194            '\u{0303}' => "\\tilde".to_string(),
1195            '\u{0304}' => "\\bar".to_string(),
1196            '\u{0306}' => "\\breve".to_string(),
1197            '\u{0307}' => "\\dot".to_string(),
1198            '\u{0308}' => "\\ddot".to_string(),
1199            '\u{030A}' => "\\mathring".to_string(),
1200            '\u{030B}' => "\\H".to_string(),
1201            '\u{030C}' => "\\check".to_string(),
1202            '\u{0327}' => "\\c".to_string(),
1203            _ => format!("\\char\"{:X}", ch as u32),
1204        },
1205        Mode::Text => match ch {
1206            '\u{0300}' => "\\`".to_string(),
1207            '\u{0301}' => "\\'".to_string(),
1208            '\u{0302}' => "\\^".to_string(),
1209            '\u{0303}' => "\\~".to_string(),
1210            '\u{0304}' => "\\=".to_string(),
1211            '\u{0306}' => "\\u".to_string(),
1212            '\u{0307}' => "\\.".to_string(),
1213            '\u{0308}' => "\\\"".to_string(),
1214            '\u{030A}' => "\\r".to_string(),
1215            '\u{030B}' => "\\H".to_string(),
1216            '\u{030C}' => "\\v".to_string(),
1217            '\u{0327}' => "\\c".to_string(),
1218            _ => format!("\\char\"{:X}", ch as u32),
1219        },
1220    }
1221}
1222
1223fn is_valid_unit(unit: &str) -> bool {
1224    matches!(
1225        unit,
1226        "pt" | "mm" | "cm" | "in" | "bp" | "pc" | "dd" | "cc" | "nd" | "nc" | "sp" | "px"
1227            | "ex" | "em" | "mu"
1228    )
1229}
1230
1231/// If the whole expression is wrapped in TeX inline/display math delimiters, parse the inside only.
1232/// The parser already runs in math mode; a leading `$` would otherwise hit the `$` / `\\(` "switch to math"
1233/// handler, which is disallowed in math mode (see `functions::math`).
1234fn strip_outer_math_delimiters(input: &str) -> &str {
1235    let s = input.trim();
1236    if s.len() >= 4 && s.starts_with("$$") && s.ends_with("$$") {
1237        return s[2..s.len() - 2].trim();
1238    }
1239    if s.len() >= 2 && s.starts_with('$') && s.ends_with('$') {
1240        return s[1..s.len() - 1].trim();
1241    }
1242    s
1243}
1244
1245/// Convenience function: parse a LaTeX string and return the AST.
1246pub fn parse(input: &str) -> ParseResult<Vec<ParseNode>> {
1247    Parser::new(strip_outer_math_delimiters(input)).parse()
1248}
1249
1250#[cfg(test)]
1251mod tests {
1252    use super::*;
1253
1254    #[test]
1255    fn test_parse_single_char() {
1256        let result = parse("x").unwrap();
1257        assert_eq!(result.len(), 1);
1258        assert_eq!(result[0].type_name(), "mathord");
1259    }
1260
1261    #[test]
1262    fn test_parse_strips_outer_dollar_inline_math() {
1263        let inner = r"C_p[\ce{H2O(l)}] = \pu{75.3 J // mol K}";
1264        let wrapped = format!("${inner}$");
1265        let a = parse(&wrapped).expect("wrapped");
1266        let b = parse(inner).expect("inner");
1267        assert_eq!(a.len(), b.len());
1268        for (x, y) in a.iter().zip(b.iter()) {
1269            assert_eq!(x.type_name(), y.type_name());
1270        }
1271    }
1272
1273    #[test]
1274    fn test_parse_addition() {
1275        let result = parse("a+b").unwrap();
1276        assert_eq!(result.len(), 3);
1277        assert_eq!(result[0].type_name(), "mathord"); // a
1278        assert_eq!(result[1].type_name(), "atom"); // +
1279        assert_eq!(result[2].type_name(), "mathord"); // b
1280    }
1281
1282    #[test]
1283    fn test_parse_superscript() {
1284        let result = parse("x^2").unwrap();
1285        assert_eq!(result.len(), 1);
1286        assert_eq!(result[0].type_name(), "supsub");
1287    }
1288
1289    #[test]
1290    fn test_parse_subscript() {
1291        let result = parse("a_i").unwrap();
1292        assert_eq!(result.len(), 1);
1293        assert_eq!(result[0].type_name(), "supsub");
1294    }
1295
1296    #[test]
1297    fn test_parse_supsub() {
1298        let result = parse("x^2_i").unwrap();
1299        assert_eq!(result.len(), 1);
1300        assert_eq!(result[0].type_name(), "supsub");
1301        if let ParseNode::SupSub { sup, sub, .. } = &result[0] {
1302            assert!(sup.is_some());
1303            assert!(sub.is_some());
1304        } else {
1305            panic!("Expected SupSub");
1306        }
1307    }
1308
1309    #[test]
1310    fn test_parse_group() {
1311        let result = parse("{a+b}").unwrap();
1312        assert_eq!(result.len(), 1);
1313        assert_eq!(result[0].type_name(), "ordgroup");
1314    }
1315
1316    #[test]
1317    fn test_parse_frac() {
1318        let result = parse("\\frac{a}{b}").unwrap();
1319        assert_eq!(result.len(), 1);
1320        assert_eq!(result[0].type_name(), "genfrac");
1321    }
1322
1323    #[test]
1324    fn test_parse_sqrt() {
1325        let result = parse("\\sqrt{x}").unwrap();
1326        assert_eq!(result.len(), 1);
1327        assert_eq!(result[0].type_name(), "sqrt");
1328    }
1329
1330    #[test]
1331    fn test_parse_sqrt_optional() {
1332        let result = parse("\\sqrt[3]{x}").unwrap();
1333        assert_eq!(result.len(), 1);
1334        if let ParseNode::Sqrt { index, .. } = &result[0] {
1335            assert!(index.is_some());
1336        } else {
1337            panic!("Expected Sqrt");
1338        }
1339    }
1340
1341    #[test]
1342    fn test_parse_nested() {
1343        let result = parse("\\frac{\\sqrt{a^2+b^2}}{c}").unwrap();
1344        assert_eq!(result.len(), 1);
1345        assert_eq!(result[0].type_name(), "genfrac");
1346    }
1347
1348    #[test]
1349    fn test_parse_empty() {
1350        let result = parse("").unwrap();
1351        assert_eq!(result.len(), 0);
1352    }
1353
1354    #[test]
1355    fn test_parse_double_superscript_error() {
1356        let result = parse("x^2^3");
1357        assert!(result.is_err());
1358    }
1359
1360    #[test]
1361    fn test_parse_unclosed_brace_error() {
1362        let result = parse("{x");
1363        assert!(result.is_err());
1364    }
1365
1366    #[test]
1367    fn test_parse_json_output() {
1368        let result = parse("x^2").unwrap();
1369        let json = serde_json::to_string_pretty(&result).unwrap();
1370        assert!(json.contains("supsub"));
1371    }
1372}
ratex_parser/parser.rs

ratex_parser/
parser.rs
