use std::rc::Rc;
use crate::ast::{AtomFamily, ColumnSeparationType, Measurement, Mode, ParseNode, StyleLevel};
use crate::environments::{
build_environment_registry, cd_row, EnvironmentContext, EnvironmentParser,
EnvironmentRegistry, EnvironmentSpec, ArrayEnvironmentOptions,
};
use crate::error::{Diagnostic, ParseError};
use crate::function_registry::{
build_function_registry, ArgType, FunctionContext, FunctionParser, FunctionRegistry,
FunctionSpec,
};
use crate::functions::{parse_size_measurement, size_scan_candidate, valid_size_unit};
use crate::lexer::{is_ascii_alphabetic, starts_with_at};
use crate::macro_definition::MacroDefinition;
use crate::macro_expander::{is_implicit_command, ExternalCommandStatus, MacroExpander};
use crate::settings::{Settings, TrustContext};
use crate::source_location::SourceLocation;
use crate::symbol_registry::{is_registered_symbol, lookup_symbol, SymbolGroup, SymbolSpec};
use crate::text_ligature::form_text_ligatures;
use crate::token::Token;
use crate::unicode_scripts::{lookup_unicode_script, supported_codepoint, UnicodeScriptKind};
use crate::unicode_symbols::{
normalize_unicode_symbol, trailing_combining_mark_start, unicode_accent_command,
};
pub struct Parser {
mode: Mode,
gullet: MacroExpander,
settings: Settings,
function_registry: FunctionRegistry,
environment_registry: EnvironmentRegistry,
next_token: Option<Token>,
leftright_depth: usize,
}
enum AtomResult {
EmitAtom(ParseNode),
SkipAtom,
}
impl Parser {
pub(crate) fn new(
input: &str,
settings: Settings,
extra_specs: &[FunctionSpec],
extra_env_specs: &[EnvironmentSpec],
) -> Parser {
let registry = build_function_registry(extra_specs);
let command_registry = registry.clone();
let settings_for_reporter = settings.clone();
let gullet = MacroExpander::new(
input,
settings.clone(),
Rc::new(move |error_code, error_message| {
settings_for_reporter.report_nonstrict(error_code, error_message, None)
}),
Rc::new(move |name| match command_registry.get(name) {
Some(spec) => {
if spec.is_expandable() {
ExternalCommandStatus::ExternalExpandable
} else {
ExternalCommandStatus::ExternalUnexpandable
}
}
None => {
if is_registered_symbol(name) {
ExternalCommandStatus::ExternalUnexpandable
} else {
ExternalCommandStatus::ExternalUndefined
}
}
}),
);
Parser {
mode: Mode::Math,
gullet,
settings,
function_registry: registry,
environment_registry: build_environment_registry(extra_env_specs),
next_token: None,
leftright_depth: 0,
}
}
fn fetch(&mut self) -> Result<Token, ParseError> {
match &self.next_token {
Some(token) => Ok(token.clone()),
None => {
let token = self.gullet.expand_next_token()?;
self.next_token = Some(token.clone());
Ok(token)
}
}
}
fn consume(&mut self) {
self.next_token = None;
}
fn expect(&mut self, text: &str, consume: bool) -> Result<(), ParseError> {
let token = self.fetch()?;
if token.text != text {
return Err(ParseError::ExpectedToken {
expected: text.to_string(),
actual: Diagnostic::from_token(&token),
});
}
if consume {
self.consume();
}
Ok(())
}
fn parse(&mut self) -> Result<Vec<ParseNode>, ParseError> {
if !self.settings.global_group {
self.gullet.begin_group();
}
if self.settings.color_is_text_color {
self.gullet.macros.set(
"\\color".to_string(),
Some(MacroDefinition::text("\\textcolor")),
false,
);
}
let result: Result<Vec<ParseNode>, ParseError> = (|| {
let body = self.parse_expression(false, None)?;
self.expect("EOF", true)?;
Ok(body)
})();
let close_result = if self.settings.global_group {
Ok(())
} else {
self.gullet.end_group()
};
self.gullet.end_groups();
if self.settings.global_group {
self.persist_user_macros();
}
unwrap_captured(close_result, result)
}
fn persist_user_macros(&mut self) {
let user_entries = self.gullet.macros.get_user_entries();
if let Some(macros) = &mut self.settings.macro_store {
for (name, definition) in user_entries {
macros.0.insert(name, definition.clone());
}
}
}
fn subparse(&mut self, tokens: Vec<Token>) -> Result<Vec<ParseNode>, ParseError> {
let old_token = self.next_token.clone();
self.consume();
self.gullet.push_token(Token::new("}", None));
self.gullet.push_tokens(tokens);
let result = self.subparse_inner();
self.next_token = old_token;
result
}
fn subparse_inner(&mut self) -> Result<Vec<ParseNode>, ParseError> {
let body = self.parse_expression(false, Some("}"))?;
self.expect("}", true)?;
Ok(body)
}
fn parse_math_mode(&mut self, close: &str) -> Result<Vec<ParseNode>, ParseError> {
let outer_mode = self.mode;
self.switch_mode(Mode::Math);
let result = self.parse_math_mode_inner(close);
self.switch_mode(outer_mode);
result
}
fn parse_math_mode_inner(&mut self, close: &str) -> Result<Vec<ParseNode>, ParseError> {
let body = self.parse_expression(false, Some(close))?;
self.expect(close, true)?;
Ok(body)
}
fn current_color(&self) -> Result<Option<String>, ParseError> {
match self.gullet.macros.get("\\current@color") {
None => Ok(None),
Some(MacroDefinition::Text(color)) => Ok(Some(color.clone())),
Some(MacroDefinition::Expansion(_)) => Err(ParseError::InvalidArgument {
message: "\\current@color set to non-string in \\right".to_string(),
loc: None,
}),
}
}
fn parse_left_right(&mut self, left: &str) -> Result<ParseNode, ParseError> {
self.leftright_depth += 1;
let result = self.parse_left_right_inner(left);
self.leftright_depth -= 1;
result
}
fn parse_left_right_inner(&mut self, left: &str) -> Result<ParseNode, ParseError> {
let body = self.parse_expression(false, None)?;
self.expect("\\right", false)?;
let Some(AtomResult::EmitAtom(ParseNode::LeftRightRight {
delim: right,
color,
..
})) = self.parse_function(None, None)?
else {
return Err(ParseError::InternalInvariant {
message: "\\right did not produce a closing delimiter".to_string(),
});
};
Ok(ParseNode::LeftRight {
mode: self.mode,
body,
left: left.to_string(),
right,
right_color: color,
})
}
fn parse_expression(
&mut self,
break_on_infix: bool,
break_on_token_text: Option<&str>,
) -> Result<Vec<ParseNode>, ParseError> {
let mut body: Vec<ParseNode> = Vec::new();
loop {
if self.mode == Mode::Math {
self.consume_spaces()?;
}
let token = self.fetch()?;
if self.should_break_expression(&token.text, break_on_infix, break_on_token_text) {
return self.finish_expression(body);
}
match self.parse_atom(break_on_token_text)? {
None => return self.finish_expression(body),
Some(AtomResult::SkipAtom) => continue,
Some(AtomResult::EmitAtom(node)) => {
body.push(node);
continue;
}
}
}
}
fn should_break_expression(
&self,
text: &str,
break_on_infix: bool,
break_on_token_text: Option<&str>,
) -> bool {
is_end_of_expression(text)
|| (break_on_token_text.is_some_and(|stop| text == stop))
|| (break_on_infix
&& self
.function_registry
.get(text)
.is_some_and(|spec| spec.infix))
}
fn finish_expression(&mut self, body: Vec<ParseNode>) -> Result<Vec<ParseNode>, ParseError> {
let normalized = if self.mode == Mode::Text {
form_text_ligatures(body)
} else {
body
};
self.handle_infix_nodes(normalized)
}
fn consume_spaces(&mut self) -> Result<(), ParseError> {
loop {
if self.fetch()?.text != " " {
break;
}
self.consume();
}
Ok(())
}
fn parse_atom(&mut self, break_on_token_text: Option<&str>) -> Result<Option<AtomResult>, ParseError> {
match self.parse_group("atom", break_on_token_text)? {
None => Ok(None),
Some(AtomResult::SkipAtom) => Ok(Some(AtomResult::SkipAtom)),
Some(AtomResult::EmitAtom(ParseNode::Internal { .. })) => Ok(Some(AtomResult::SkipAtom)),
Some(AtomResult::EmitAtom(base)) if self.mode == Mode::Text => {
Ok(Some(AtomResult::EmitAtom(base)))
}
Some(AtomResult::EmitAtom(base)) => {
Ok(Some(AtomResult::EmitAtom(self.parse_scripts(base)?)))
}
}
}
fn parse_scripts(&mut self, base: ParseNode) -> Result<ParseNode, ParseError> {
let mut base = base;
let mut sup: Option<ParseNode> = None;
let mut sub: Option<ParseNode> = None;
loop {
self.consume_spaces()?;
let token = self.fetch()?;
if token.text == "\\limits" || token.text == "\\nolimits" {
base = set_limits(base, token.text == "\\limits", token.loc.clone())?;
self.consume();
continue;
} else if token.text == "^" {
if sup.is_some() {
return Err(ParseError::DoubleSuperscript { loc: token.loc.clone() });
}
sup = Some(self.handle_sup_subscript("superscript")?);
continue;
} else if token.text == "_" {
if sub.is_some() {
return Err(ParseError::DoubleSubscript { loc: token.loc.clone() });
}
sub = Some(self.handle_sup_subscript("subscript")?);
continue;
} else if token.text == "'" {
if sup.is_some() {
return Err(ParseError::DoubleSuperscript { loc: token.loc.clone() });
}
sup = Some(self.parse_prime_run()?);
continue;
} else {
match lookup_unicode_script(&token.text) {
None => return Ok(make_supsub_or_base(self.mode, base, sup, sub)),
Some(first_script) => {
let (is_subscript, script_tokens) =
self.consume_unicode_script_run(first_script)?;
let body = self.subparse(script_tokens)?;
let group = ParseNode::OrdGroup {
mode: Mode::Math,
loc: None,
body,
semisimple: false,
};
if is_subscript {
sub = Some(group);
} else {
sup = Some(group);
}
continue;
}
}
}
}
}
fn parse_prime_run(&mut self) -> Result<ParseNode, ParseError> {
let mut primes: Vec<ParseNode> = Vec::new();
while self.fetch()?.text == "'" {
let prime_token = self.fetch()?;
primes.push(ParseNode::TextOrd {
mode: self.mode,
loc: prime_token.loc.clone(),
text: "\\prime".to_string(),
});
self.consume();
}
if self.fetch()?.text == "^" {
primes.push(self.handle_sup_subscript("superscript")?);
}
Ok(ParseNode::OrdGroup {
mode: self.mode,
loc: None,
body: primes,
semisimple: false,
})
}
fn consume_unicode_script_run(
&mut self,
first: &crate::unicode_scripts::UnicodeScript,
) -> Result<(bool, Vec<Token>), ParseError> {
let is_subscript = first.kind == UnicodeScriptKind::UnicodeSubscript;
let mut tokens: Vec<Token> = vec![Token::new(first.replacement.clone(), None)];
self.consume();
loop {
let next = self.fetch()?;
match lookup_unicode_script(&next.text) {
Some(script)
if (script.kind == UnicodeScriptKind::UnicodeSubscript) == is_subscript =>
{
tokens.push(Token::new(script.replacement.clone(), None));
self.consume();
continue;
}
_ => {
tokens.reverse();
return Ok((is_subscript, tokens));
}
}
}
}
fn handle_sup_subscript(&mut self, name: &str) -> Result<ParseNode, ParseError> {
let token = self.fetch()?;
self.consume();
self.consume_spaces()?;
loop {
match self.parse_group(name, None)? {
Some(AtomResult::EmitAtom(ParseNode::Internal { .. })) | Some(AtomResult::SkipAtom) => {
continue;
}
Some(AtomResult::EmitAtom(group)) => return Ok(group),
None => {
return Err(ParseError::ExpectedGroupAfter {
symbol: token.text.clone(),
loc: token.loc.clone(),
})
}
}
}
}
fn parse_group(
&mut self,
name: &str,
break_on_token_text: Option<&str>,
) -> Result<Option<AtomResult>, ParseError> {
let first_token = self.fetch()?;
let text = first_token.text.clone();
if text == "{" || text == "\\begingroup" {
Ok(Some(AtomResult::EmitAtom(
self.parse_group_body(&first_token, &text)?,
)))
} else {
match self.parse_function(break_on_token_text, Some(name))? {
Some(result) => Ok(Some(result)),
None => match self.parse_symbol()? {
Some(node) => Ok(Some(AtomResult::EmitAtom(node))),
None => self.handle_undefined_control(&first_token),
},
}
}
}
fn parse_group_body(&mut self, first_token: &Token, text: &str) -> Result<ParseNode, ParseError> {
self.consume();
let group_end = if text == "{" { "}" } else { "\\endgroup" };
self.gullet.begin_group();
let body = self.parse_expression(false, Some(group_end))?;
let last = self.fetch()?;
self.expect(group_end, true)?;
self.gullet.end_group()?;
let loc = match (&first_token.loc, &last.loc) {
(Some(start_loc), Some(end_loc)) => Some(SourceLocation::range(start_loc, end_loc)),
_ => None,
};
Ok(ParseNode::OrdGroup {
mode: self.mode,
loc,
body,
semisimple: text == "\\begingroup",
})
}
fn handle_undefined_control(
&mut self,
token: &Token,
) -> Result<Option<AtomResult>, ParseError> {
let text = token.text.clone();
if !is_undefined_control_sequence(&text) {
return Ok(None);
}
if !self.settings.throw_on_error {
self.consume();
return Ok(Some(AtomResult::EmitAtom(format_unsupported_command(
self.mode,
&self.settings,
&text,
))));
}
Err(ParseError::UndefinedControlSequence {
name: text,
loc: token.loc.clone(),
})
}
fn parse_function(
&mut self,
break_on_token_text: Option<&str>,
name: Option<&str>,
) -> Result<Option<AtomResult>, ParseError> {
let token = self.fetch()?;
let func_data = match self.function_registry.get(&token.text) {
None => return Ok(None),
Some(fd) => fd.clone(),
};
self.consume();
if let Some(context_name) = name
&& context_name != "atom" && !func_data.allowed_in_argument {
return Err(ParseError::FunctionNotAllowed {
func_name: token.text.clone(),
context: context_name.to_string(),
loc: token.loc.clone(),
});
}
if self.mode == Mode::Text && !func_data.allowed_in_text {
return Err(ParseError::FunctionNotAllowed {
func_name: token.text.clone(),
context: "text mode".to_string(),
loc: token.loc.clone(),
});
}
if self.mode == Mode::Math && !func_data.allowed_in_math {
return Err(ParseError::FunctionNotAllowed {
func_name: token.text.clone(),
context: "math mode".to_string(),
loc: token.loc.clone(),
});
}
let (args, opt_args) = self.parse_arguments(&token.text, &func_data)?;
Ok(Some(AtomResult::EmitAtom(self.call_function(
&token.text,
args,
opt_args,
Some(token.clone()),
break_on_token_text.map(|s| s.to_string()),
)?)))
}
fn call_function(
&mut self,
func_name: &str,
args: Vec<ParseNode>,
opt_args: Vec<Option<ParseNode>>,
token: Option<Token>,
break_on_token_text: Option<String>,
) -> Result<ParseNode, ParseError> {
let context = FunctionContext {
func_name: func_name.to_string(),
mode: self.mode,
token: token.clone(),
break_on_token_text: break_on_token_text.clone(),
display_mode: self.settings.display_mode,
};
let handler = {
let spec = self.function_registry.get(func_name).ok_or_else(|| {
ParseError::MissingFunctionHandler {
func_name: func_name.to_string(),
loc: None,
}
})?;
spec.handler.ok_or_else(|| ParseError::MissingFunctionHandler {
func_name: func_name.to_string(),
loc: None,
})?
};
handler(self, &context, &args, &opt_args)
}
fn parse_arguments(
&mut self,
func: &str,
func_data: &FunctionSpec,
) -> Result<(Vec<ParseNode>, Vec<Option<ParseNode>>), ParseError> {
let mut args: Vec<ParseNode> = Vec::new();
let mut opt_args: Vec<Option<ParseNode>> = Vec::new();
let total_args = func_data.num_args + func_data.num_optional_args;
let mut index = 0;
while index < total_args {
let optional = index < func_data.num_optional_args;
let arg_type = match func_data.arg_types.get(index) {
Some(kind) => *kind,
None if func_data.primitive => ArgType::PrimitiveArg,
None
if func_data
.primitive_after_missing_optional
.is_some_and(|optional_index| {
index == func_data.num_optional_args
&& opt_args
.get(optional_index)
.is_some_and(|value| value.is_none())
}) =>
{
ArgType::PrimitiveArg
}
None => ArgType::OriginalArg,
};
match self.parse_group_of_type(&format!("argument to '{func}'"), arg_type, optional)? {
None if optional => {
opt_args.push(None);
index += 1;
continue;
}
None => {
return Err(ParseError::InternalInvariant {
message: "Null mandatory function argument after parser validation"
.to_string(),
})
}
Some(arg) if optional => {
opt_args.push(Some(arg));
index += 1;
continue;
}
Some(arg) => {
args.push(arg);
index += 1;
continue;
}
}
}
Ok((args, opt_args))
}
fn parse_group_of_type(
&mut self,
name: &str,
arg_type: ArgType,
optional: bool,
) -> Result<Option<ParseNode>, ParseError> {
match arg_type {
ArgType::ColorArg => self.parse_color_group(optional),
ArgType::SizeArg => self.parse_size_group(optional),
ArgType::UrlArg => self.parse_url_group(optional),
ArgType::RawArg => Ok(self
.parse_string_group(optional)?
.map(|token| ParseNode::Raw {
mode: Mode::Text,
string: token.text,
})),
ArgType::MathArg => self.parse_argument_group(optional, Some(Mode::Math)),
ArgType::TextArg => self.parse_argument_group(optional, Some(Mode::Text)),
ArgType::HboxArg => Ok(self.parse_argument_group(optional, Some(Mode::Text))?.map(
|group| ParseNode::Styling {
mode: group.mode(),
body: vec![group],
style: StyleLevel::TextStyle,
reset_font: true,
},
)),
ArgType::PrimitiveArg => self.parse_primitive_group(name, optional),
ArgType::OriginalArg => self.parse_argument_group(optional, None),
}
}
fn parse_primitive_group(&mut self, name: &str, optional: bool) -> Result<Option<ParseNode>, ParseError> {
if optional {
return Err(ParseError::InvalidArgument {
message: "A primitive argument cannot be optional".to_string(),
loc: None,
});
}
let Some(AtomResult::EmitAtom(group)) = self.parse_group(name, None)? else {
let token = self.fetch()?;
return Err(ParseError::InvalidArgument {
message: format!("Expected group as {name}"),
loc: token.loc,
});
};
Ok(Some(group))
}
fn parse_argument_group(
&mut self,
optional: bool,
mode: Option<Mode>,
) -> Result<Option<ParseNode>, ParseError> {
match self.gullet.scan_argument(optional)? {
None => Ok(None),
Some(arg_token) => {
let outer_mode = self.mode;
if let Some(argument_mode) = mode {
self.switch_mode(argument_mode);
}
self.gullet.begin_group();
let result: Result<ParseNode, ParseError> = (|| {
let body = self.parse_expression(false, Some("EOF"))?;
self.expect("EOF", true)?;
Ok(ParseNode::OrdGroup {
mode: self.mode,
loc: arg_token.loc.clone(),
body,
semisimple: false,
})
})();
let close_result = self.gullet.end_group();
self.switch_mode(outer_mode);
Ok(Some(unwrap_captured(close_result, result)?))
}
}
}
fn parse_string_group(&mut self, optional: bool) -> Result<Option<Token>, ParseError> {
match self.gullet.scan_argument(optional)? {
None => Ok(None),
Some(mut arg_token) => {
let mut builder = String::new();
loop {
let token = self.fetch()?;
if token.text == "EOF" {
self.consume();
arg_token.text = builder;
break Ok(Some(arg_token));
}
builder.push_str(&token.text);
self.consume();
}
}
}
}
fn parse_color_group(&mut self, optional: bool) -> Result<Option<ParseNode>, ParseError> {
match self.parse_string_group(optional)? {
None => Ok(None),
Some(token) => {
let color = normalized_color(&token.text).ok_or_else(|| {
ParseError::InvalidArgument {
message: format!("Invalid color: '{}'", token.text),
loc: token.loc.clone(),
}
})?;
Ok(Some(ParseNode::ColorToken {
mode: self.mode,
color,
}))
}
}
}
fn parse_url_group(&mut self, optional: bool) -> Result<Option<ParseNode>, ParseError> {
self.gullet.set_lexer_catcode("%", 13);
self.gullet.set_lexer_catcode("~", 12);
let parsed = self.parse_string_group(optional);
self.gullet.set_lexer_catcode("%", 14);
self.gullet.set_lexer_catcode("~", 13);
match parsed? {
None => Ok(None),
Some(token) => Ok(Some(ParseNode::Url {
mode: self.mode,
url: unescape_url(&token.text),
})),
}
}
fn parse_size_regex_group(&mut self) -> Result<Token, ParseError> {
let first_token = self.fetch()?;
let mut last_token = first_token.clone();
let mut builder = String::new();
loop {
let token = self.fetch()?;
if token.text == "EOF" {
break;
}
let candidate = format!("{builder}{}", token.text);
if !size_scan_candidate(&candidate) {
break;
}
builder.push_str(&token.text);
last_token = token;
self.consume();
}
if builder.is_empty() {
return Err(ParseError::InvalidArgument {
message: format!("Invalid size: '{}'", first_token.text),
loc: first_token.loc.clone(),
});
}
Ok(first_token.range(&last_token, builder))
}
fn parse_size_group(&mut self, optional: bool) -> Result<Option<ParseNode>, ParseError> {
self.gullet.consume_spaces()?;
let parsed = if !optional && self.gullet.future()?.text != "{" {
Some(self.parse_size_regex_group()?)
} else {
self.parse_string_group(optional)?
};
match parsed {
None => Ok(None),
Some(token) => {
let mut text = token.text;
let is_blank = !optional && text.is_empty();
if is_blank {
text = "0pt".to_string();
}
let value = parse_size_measurement(&text)?.ok_or_else(|| {
ParseError::InvalidArgument {
message: format!("Invalid size: '{text}'"),
loc: token.loc.clone(),
}
})?;
if !valid_size_unit(&value.unit) {
return Err(ParseError::InvalidArgument {
message: format!("Invalid unit: '{}'", value.unit),
loc: token.loc.clone(),
});
}
Ok(Some(ParseNode::Size {
mode: self.mode,
value,
is_blank,
}))
}
}
}
fn handle_infix_nodes(&mut self, body: Vec<ParseNode>) -> Result<Vec<ParseNode>, ParseError> {
let mut infix: Option<(usize, String)> = None;
for (index, node) in body.iter().enumerate() {
if let ParseNode::Infix { replace_with, loc, .. } = node {
if infix.is_some() {
return Err(ParseError::InvalidArgument {
message: "only one infix operator per group".to_string(),
loc: loc.clone(),
});
}
infix = Some((index, replace_with.clone()));
}
}
if let Some((index, func_name)) = infix {
let numer = infix_side_group(self.mode, body[..index].to_vec());
let denom = infix_side_group(self.mode, body[index + 1..].to_vec());
let node = if func_name == "\\\\abovefrac" {
self.call_function(
&func_name,
vec![numer, body[index].clone(), denom],
Vec::new(),
None,
None,
)?
} else {
self.call_function(&func_name, vec![numer, denom], Vec::new(), None, None)?
};
Ok(vec![node])
} else {
Ok(body)
}
}
fn parse_environment(&mut self, name: &str) -> Result<ParseNode, ParseError> {
let spec = self
.environment_registry
.get(name)
.cloned()
.ok_or_else(|| ParseError::InvalidArgument {
message: format!("No such environment: {name}"),
loc: None,
})?;
let arguments = FunctionSpec {
names: Vec::new(),
num_args: spec.num_args,
num_optional_args: spec.num_optional_args,
arg_types: spec.arg_types.clone(),
..Default::default()
};
let (args, opt_args) = self.parse_arguments(&format!("\\begin{{{name}}}"), &arguments)?;
let context = EnvironmentContext {
mode: self.mode,
display_mode: self.settings.display_mode,
leqno: self.settings.leqno,
env_name: name.to_string(),
};
let result = (spec.handler)(self, &context, &args, &opt_args)?;
self.expect("\\end", false)?;
match self.parse_function(None, None)? {
Some(AtomResult::EmitAtom(ParseNode::EnvironmentEnd { name: end_name, .. }))
if end_name == name =>
{
Ok(result)
}
Some(AtomResult::EmitAtom(ParseNode::EnvironmentEnd { name: end_name, .. })) => {
Err(ParseError::InvalidArgument {
message: format!(
"Mismatch: \\begin{{{name}}} matched by \\end{{{end_name}}}"
),
loc: None,
})
}
_ => Err(ParseError::InternalInvariant {
message: "Expected environment end".to_string(),
}),
}
}
fn parse_matrix_alignment(&mut self) -> Result<Option<String>, ParseError> {
self.consume_spaces()?;
if self.fetch()?.text != "[" {
return Ok(None);
}
self.consume();
self.consume_spaces()?;
let token = self.fetch()?;
if token.text != "l" && token.text != "c" && token.text != "r" {
return Err(ParseError::InvalidArgument {
message: "Expected l or c or r".to_string(),
loc: token.loc.clone(),
});
}
self.consume();
self.consume_spaces()?;
self.expect("]", true)?;
Ok(Some(token.text))
}
fn parse_prefixed_function(&mut self, name: &str) -> Result<ParseNode, ParseError> {
self.gullet.push_token(Token::new(name, None));
let Some(AtomResult::EmitAtom(node)) = self.parse_function(None, None)? else {
return Err(ParseError::InternalInvariant {
message: "Expected function after macro prefix".to_string(),
});
};
Ok(node)
}
fn parse_symbol(&mut self) -> Result<Option<ParseNode>, ParseError> {
let token = self.fetch()?;
let original_text = token.text.clone();
if original_text == "EOF"
|| original_text == "^"
|| original_text == "_"
|| original_text == "{"
|| original_text == "}"
|| original_text == "&"
{
Ok(None)
} else if is_verb_token(&original_text) {
self.consume();
Ok(Some(parse_verb_token(&original_text)?))
} else {
self.parse_symbol_text(
&token,
&original_text,
normalize_unicode_symbol(self.mode, &original_text),
)
}
}
fn parse_symbol_text(
&mut self,
token: &Token,
original_text: &str,
normalized: String,
) -> Result<Option<ParseNode>, ParseError> {
if self.mode == Mode::Math && normalized != original_text {
let first = original_text.chars().next().unwrap();
self.settings.report_nonstrict(
"unicodeTextInMathMode",
&format!("Accented Unicode text character \"{first}\" used in math mode"),
Some(token),
)?;
}
let (text, marks) = split_combining_marks(&normalized);
match lookup_symbol(self.mode, &text) {
Some(spec) => {
if self.mode == Mode::Math && is_extra_latin(&text) {
let first = text.chars().next().unwrap();
self.settings.report_nonstrict(
"unicodeTextInMathMode",
&format!("Latin-1/Unicode text character \"{first}\" used in math mode"),
Some(token),
)?;
}
self.consume();
let base = make_symbol_node(self.mode, &text, token.loc.clone(), spec);
match marks {
None => Ok(Some(base)),
Some(accents) => Ok(Some(apply_unicode_accents(
self.mode,
token.loc.clone(),
base,
&accents,
)?)),
}
}
None if is_non_ascii(&text) => {
let first = text.chars().next().unwrap();
if !supported_codepoint(first as u32) {
self.settings.report_nonstrict(
"unknownSymbol",
&format!(
"Unrecognized Unicode character \"{first}\" ({})",
first as u32
),
Some(token),
)?;
} else if self.mode == Mode::Math {
self.settings.report_nonstrict(
"unicodeTextInMathMode",
&format!("Unicode text character \"{first}\" used in math mode"),
Some(token),
)?;
}
self.consume();
Ok(Some(ParseNode::TextOrd {
mode: Mode::Text,
loc: token.loc.clone(),
text,
}))
}
None => Ok(None),
}
}
fn switch_mode(&mut self, mode: Mode) {
if self.mode != mode {
self.mode = mode;
self.gullet.switch_mode(mode);
}
}
fn consume_array_hlines(&mut self) -> Result<Vec<bool>, ParseError> {
let mut lines: Vec<bool> = Vec::new();
self.consume_spaces()?;
while self.fetch()?.text == "\\hline" || self.fetch()?.text == "\\hdashline" {
let dashed = self.fetch()?.text == "\\hdashline";
self.consume();
lines.push(dashed);
self.consume_spaces()?;
}
Ok(lines)
}
fn take_array_tag(
&mut self,
auto_tag: Option<bool>,
) -> Result<(Option<Vec<ParseNode>>, bool), ParseError> {
let Some(automatic) = auto_tag else {
return Ok((None, false));
};
if self.gullet.macros.get("\\df@tag").is_none() {
return Ok((None, automatic));
}
let tag = self.subparse(vec![Token::new("\\df@tag", None)])?;
self.gullet.macros.set("\\df@tag".to_string(), None, true);
Ok((Some(tag), false))
}
fn push_array_tag(
&mut self,
tags: &mut Vec<Option<Vec<ParseNode>>>,
auto_tags: &mut Vec<bool>,
auto_tag: Option<bool>,
) -> Result<(), ParseError> {
let (tag, automatic) = self.take_array_tag(auto_tag)?;
if auto_tag.is_some() {
tags.push(tag);
auto_tags.push(automatic);
}
Ok(())
}
fn parse_array_row_gap(&mut self) -> Result<Option<Measurement>, ParseError> {
if self.gullet.future()?.text == " " {
Ok(None)
} else {
match self.parse_size_group(true)? {
Some(ParseNode::Size { value, .. }) => Ok(Some(value)),
Some(_) => Err(ParseError::InternalInvariant {
message: "Expected array row gap".to_string(),
}),
None => Ok(None),
}
}
}
fn parse_array_environment(
&mut self,
options: ArrayEnvironmentOptions,
) -> Result<ParseNode, ParseError> {
self.gullet.begin_group();
self.gullet
.macros
.set("\\cr".to_string(), Some(MacroDefinition::text("\\\\\\relax")), false);
self.gullet.begin_group();
let result: Result<ParseNode, ParseError> = (|| {
let mut body: Vec<Vec<ParseNode>> = vec![Vec::new()];
let mut row_gaps: Vec<Option<Measurement>> = Vec::new();
let mut hlines_before_row: Vec<Vec<bool>> = vec![self.consume_array_hlines()?];
let mut tags: Vec<Option<Vec<ParseNode>>> = Vec::new();
let mut auto_tags: Vec<bool> = Vec::new();
loop {
let cell_body = self.parse_expression(false, Some("\\\\"))?;
let cell = ParseNode::Styling {
mode: self.mode,
body: vec![ParseNode::OrdGroup {
mode: self.mode,
loc: None,
body: cell_body,
semisimple: false,
}],
style: options.cell_style,
reset_font: true,
};
self.gullet.end_group()?;
self.gullet.begin_group();
let Some(row) = body.last_mut() else {
return Err(ParseError::InternalInvariant {
message: "Missing array row".to_string(),
});
};
row.push(cell);
let text = self.fetch()?.text;
match text.as_str() {
"&" => {
if array_row_at_max(&body, options.max_columns) {
return Err(ParseError::InvalidArgument {
message: "Too many tab characters: &".to_string(),
loc: None,
});
}
self.consume();
}
"\\end" => {
self.push_array_tag(&mut tags, &mut auto_tags, options.auto_tag)?;
break;
}
"\\\\" => {
if options.single_row {
return Err(ParseError::InvalidArgument {
message: "Expected \\end".to_string(),
loc: None,
});
}
self.consume();
row_gaps.push(self.parse_array_row_gap()?);
self.push_array_tag(&mut tags, &mut auto_tags, options.auto_tag)?;
hlines_before_row.push(self.consume_array_hlines()?);
body.push(Vec::new());
}
_ => {
return Err(ParseError::InvalidArgument {
message: format!("Expected & or \\\\ or \\end, got {text}"),
loc: None,
})
}
}
}
if hlines_before_row.len() < body.len() + 1 {
hlines_before_row.push(Vec::new());
}
Ok(ParseNode::Array {
mode: self.mode,
body,
add_jot: options.add_jot,
array_stretch: options.array_stretch,
columns: options.columns.clone(),
row_gaps,
hskip_before_and_after: options.hskip_before_and_after,
hlines_before_row,
column_separation_type: options.column_separation_type,
tags: if options.auto_tag.is_some() { Some(tags) } else { None },
auto_tags: if options.auto_tag.is_some() { Some(auto_tags) } else { None },
leqno: options.leqno,
})
})();
let close_cell = self.gullet.end_group();
let close_array = self.gullet.end_group();
unwrap_array_parse_result(result, close_cell, close_array)
}
fn parse_cd_environment(&mut self) -> Result<ParseNode, ParseError> {
self.gullet.begin_group();
self.gullet
.macros
.set("\\cr".to_string(), Some(MacroDefinition::text("\\\\\\relax")), false);
self.gullet.begin_group();
let result: Result<ParseNode, ParseError> = (|| {
let mut parsed_rows: Vec<Vec<ParseNode>> = vec![Vec::new()];
loop {
let part = self.parse_expression(false, Some("\\\\"))?;
let Some(row) = parsed_rows.last_mut() else {
return Err(ParseError::InternalInvariant {
message: "Missing CD row".to_string(),
});
};
row.extend(part);
match self.fetch()?.text.as_str() {
"&" => self.consume(),
"\\\\" => {
self.consume();
parsed_rows.push(Vec::new());
}
"\\end" => break,
token => {
return Err(ParseError::InvalidArgument {
message: format!("Expected \\ or \\end, got {token}"),
loc: None,
})
}
}
}
if parsed_rows.last().is_some_and(|row| row.is_empty()) {
parsed_rows.pop();
}
let mut body: Vec<Vec<ParseNode>> = Vec::new();
for (index, row) in parsed_rows.iter().enumerate() {
body.push(cd_row(row.clone(), index % 2 == 0)?);
}
let count = body.first().map_or(0, |row| row.len());
let columns: Vec<crate::ast::ArrayColumn> = (0..count)
.map(|_| crate::ast::ArrayColumn::AlignColumn {
alignment: "c".to_string(),
pre_gap: 0.25,
post_gap: 0.25,
})
.collect();
let row_gap_count = body.len() + 1;
let hlines_before_row: Vec<Vec<bool>> =
(0..row_gap_count).map(|_| Vec::new()).collect();
Ok(ParseNode::Array {
mode: Mode::Math,
body,
add_jot: true,
array_stretch: 1.0,
columns: Some(columns),
row_gaps: vec![None],
hskip_before_and_after: false,
hlines_before_row,
column_separation_type: Some(ColumnSeparationType::CdSeparation),
tags: None,
auto_tags: None,
leqno: false,
})
})();
let close_cell = self.gullet.end_group();
let close_array = self.gullet.end_group();
unwrap_array_parse_result(result, close_cell, close_array)
}
}
impl FunctionParser for Parser {
fn report_nonstrict(
&self,
error_code: &str,
error_message: &str,
token: Option<&Token>,
) -> Result<(), ParseError> {
self.settings.report_nonstrict(error_code, error_message, token)
}
fn use_strict_behavior(
&self,
error_code: &str,
error_message: &str,
token: Option<&Token>,
) -> bool {
self.settings.use_strict_behavior(error_code, error_message, token)
}
fn is_trusted(&self, context: TrustContext) -> bool {
self.settings.is_trusted(context)
}
fn current_color(&self) -> Result<Option<String>, ParseError> {
self.current_color()
}
fn in_left_right(&self) -> bool {
self.leftright_depth > 0
}
fn is_expandable(&self, name: &str) -> bool {
self.gullet.is_expandable(name)
}
fn get_macro(&self, name: &str) -> Option<MacroDefinition> {
self.gullet.macros.get(name).cloned()
}
fn set_macro(&mut self, name: &str, definition: Option<MacroDefinition>) {
self.gullet.macros.set(name.to_string(), definition, false);
}
fn set_macro_definition(&mut self, name: &str, definition: MacroDefinition, global: bool) {
self.gullet.macros.set(name.to_string(), Some(definition), global);
}
fn parse_expression(
&mut self,
break_on_infix: bool,
break_on_token_text: Option<&str>,
) -> Result<Vec<ParseNode>, ParseError> {
self.parse_expression(break_on_infix, break_on_token_text)
}
fn parse_math_mode(&mut self, closing: &str) -> Result<Vec<ParseNode>, ParseError> {
self.parse_math_mode(closing)
}
fn parse_left_right(&mut self, open: &str) -> Result<ParseNode, ParseError> {
self.parse_left_right(open)
}
fn parse_optional_size(&mut self) -> Result<Option<Measurement>, ParseError> {
if self.gullet.future()?.text != "[" {
Ok(None)
} else {
match self.parse_size_group(true)? {
Some(ParseNode::Size { value, .. }) => Ok(Some(value)),
_ => Err(ParseError::InternalInvariant {
message: "Expected optional size".to_string(),
}),
}
}
}
fn parse_prefixed_function(&mut self, name: &str) -> Result<ParseNode, ParseError> {
self.parse_prefixed_function(name)
}
fn parse_environment(&mut self, name: &str) -> Result<ParseNode, ParseError> {
self.parse_environment(name)
}
fn pop_token(&mut self) -> Result<Token, ParseError> {
self.gullet.pop_token()
}
fn future_token(&mut self) -> Result<Token, ParseError> {
self.gullet.future()
}
fn push_token(&mut self, token: Token) {
self.gullet.push_token(token);
}
fn consume_spaces(&mut self) -> Result<(), ParseError> {
self.gullet.consume_spaces()
}
fn consume_macro_arg(&mut self) -> Result<Vec<Token>, ParseError> {
Ok(self.gullet.consume_arg(None)?.tokens)
}
fn expand_tokens(&mut self, tokens: Vec<Token>) -> Result<Vec<Token>, ParseError> {
self.gullet.expand_tokens(tokens)
}
}
impl EnvironmentParser for Parser {
fn parse_array(&mut self, options: ArrayEnvironmentOptions) -> Result<ParseNode, ParseError> {
self.parse_array_environment(options)
}
fn parse_matrix_alignment(&mut self) -> Result<Option<String>, ParseError> {
self.parse_matrix_alignment()
}
fn parse_cd(&mut self) -> Result<ParseNode, ParseError> {
self.parse_cd_environment()
}
}
pub fn parse(input: &str, settings: &mut Settings) -> Result<Vec<ParseNode>, ParseError> {
parse_with_specs(input, settings, &[], &[])
}
pub fn parse_with_specs(
input: &str,
settings: &mut Settings,
extra_specs: &[FunctionSpec],
extra_env_specs: &[EnvironmentSpec],
) -> Result<Vec<ParseNode>, ParseError> {
let mut parser = Parser::new(input, settings.clone(), extra_specs, extra_env_specs);
parser
.gullet
.macros
.set("\\df@tag".to_string(), None, false);
let parse_result = parser.parse();
if settings.global_group {
settings.macro_store = parser.settings.macro_store.clone();
}
let mut body = parse_result?;
if parser.gullet.macros.get("\\df@tag").is_some() {
if !settings.display_mode {
return Err(ParseError::InvalidArgument {
message: "\\tag works only in display equations".to_string(),
loc: None,
});
}
let tag = parser.subparse(vec![Token::new("\\df@tag", None)])?;
body = vec![ParseNode::Tag {
mode: Mode::Text,
body,
tag,
}];
}
parser
.gullet
.macros
.set("\\current@color".to_string(), None, false);
parser
.gullet
.macros
.set("\\color".to_string(), None, false);
if settings.display_mode {
Ok(vec![ParseNode::Styling {
mode: Mode::Math,
body,
style: StyleLevel::DisplayStyle,
reset_font: true,
}])
} else {
Ok(body)
}
}
fn unwrap_captured<V>(close: Result<(), ParseError>, value: Result<V, ParseError>) -> Result<V, ParseError> {
match (close, value) {
(Err(err), _) => Err(err),
(_, Err(err)) => Err(err),
(Ok(()), Ok(value)) => Ok(value),
}
}
fn is_end_of_expression(text: &str) -> bool {
text == "}" || text == "\\endgroup" || text == "\\end" || text == "\\right" || text == "&"
}
fn set_limits(base: ParseNode, limits: bool, loc: Option<SourceLocation>) -> Result<ParseNode, ParseError> {
match base {
ParseNode::Op {
mode,
parent_is_sup_sub,
suppress_base_shift,
content,
..
} => Ok(ParseNode::Op {
mode,
limits,
always_handle_sup_sub: true,
parent_is_sup_sub,
suppress_base_shift,
content,
}),
ParseNode::OperatorName {
mode,
body,
always_handle_sup_sub: true,
parent_is_sup_sub,
..
} => Ok(ParseNode::OperatorName {
mode,
body,
always_handle_sup_sub: true,
limits,
parent_is_sup_sub,
}),
_ => Err(ParseError::InvalidArgument {
message: "Limit controls must follow a math operator".to_string(),
loc,
}),
}
}
fn make_supsub_or_base(mode: Mode, base: ParseNode, sup: Option<ParseNode>, sub: Option<ParseNode>) -> ParseNode {
match (sup, sub) {
(None, None) => base,
(sup, sub) => ParseNode::SupSub {
mode,
base: Some(Box::new(base)),
sup: sup.map(Box::new),
sub: sub.map(Box::new),
},
}
}
fn infix_side_group(mode: Mode, body: Vec<ParseNode>) -> ParseNode {
if body.len() == 1 && matches!(body[0], ParseNode::OrdGroup { .. }) {
body[0].clone()
} else {
ParseNode::OrdGroup {
mode,
loc: None,
body,
semisimple: false,
}
}
}
fn is_undefined_control_sequence(text: &str) -> bool {
!text.is_empty() && text.starts_with('\\') && !is_implicit_command(text)
}
fn is_verb_token(text: &str) -> bool {
let chars: Vec<char> = text.chars().collect();
chars.len() > 5
&& starts_with_at(&chars, 0, "\\verb")
&& !is_ascii_alphabetic(chars[5])
}
fn parse_verb_token(text: &str) -> Result<ParseNode, ParseError> {
let chars: Vec<char> = text.chars().collect();
let raw_argument = &chars[5..];
let star = !raw_argument.is_empty() && raw_argument[0] == '*';
let argument: &[char] = if star { &raw_argument[1..] } else { raw_argument };
if argument.len() < 2 || argument[0] != argument[argument.len() - 1] {
Err(ParseError::InternalInvariant {
message: "\\verb assertion failed -- please report what input caused this bug"
.to_string(),
})
} else {
Ok(ParseNode::Verb {
mode: Mode::Text,
loc: None,
body: argument[1..argument.len() - 1].iter().collect(),
star,
})
}
}
fn is_extra_latin(text: &str) -> bool {
!text.is_empty() && matches!(text.chars().next(), Some('Ð' | 'Þ' | 'þ'))
}
fn is_non_ascii(text: &str) -> bool {
!text.is_empty() && text.chars().next().is_some_and(|c| c as u32 >= 0x80)
}
fn split_combining_marks(normalized: &str) -> (String, Option<String>) {
match trailing_combining_mark_start(normalized) {
None => (normalized.to_string(), None),
Some(start) => {
let chars: Vec<char> = normalized.chars().collect();
let base: String = chars[..start].iter().collect();
let base = if base == "i" {
"ı".to_string()
} else if base == "j" {
"ȷ".to_string()
} else {
base
};
let marks: String = chars[start..].iter().collect();
(base, Some(marks))
}
}
}
fn apply_unicode_accents(
mode: Mode,
loc: Option<SourceLocation>,
base: ParseNode,
accents: &str,
) -> Result<ParseNode, ParseError> {
let mut result = base;
for accent in accents.chars() {
let accent_text = accent.to_string();
let Some(label) = unicode_accent_command(mode, &accent_text) else {
return Err(ParseError::InvalidArgument {
message: format!("Unknown accent ' {accent_text}'"),
loc: loc.clone(),
});
};
result = ParseNode::Accent {
mode,
loc: loc.clone(),
label,
is_stretchy: false,
is_shifty: true,
base: Box::new(result),
};
}
Ok(result)
}
fn make_symbol_node(
mode: Mode,
text: &str,
loc: Option<SourceLocation>,
spec: &SymbolSpec,
) -> ParseNode {
let text = text.to_string();
match spec.group {
SymbolGroup::AccentTokenGroup => ParseNode::AccentToken { mode, loc, text },
SymbolGroup::BinaryGroup => ParseNode::Atom {
mode,
loc,
family: AtomFamily::Mbin,
text,
},
SymbolGroup::CloseGroup => ParseNode::Atom {
mode,
loc,
family: AtomFamily::Mclose,
text,
},
SymbolGroup::InnerGroup => ParseNode::Atom {
mode,
loc,
family: AtomFamily::Minner,
text,
},
SymbolGroup::MathOrdGroup => ParseNode::MathOrd { mode, loc, text },
SymbolGroup::OperatorTokenGroup => ParseNode::OperatorToken { mode, loc, text },
SymbolGroup::OpenGroup => ParseNode::Atom {
mode,
loc,
family: AtomFamily::Mopen,
text,
},
SymbolGroup::PunctuationGroup => ParseNode::Atom {
mode,
loc,
family: AtomFamily::Mpunct,
text,
},
SymbolGroup::RelationGroup => ParseNode::Atom {
mode,
loc,
family: AtomFamily::Mrel,
text,
},
SymbolGroup::SpacingGroup => ParseNode::Spacing { mode, loc, text },
SymbolGroup::TextOrdGroup => ParseNode::TextOrd { mode, loc, text },
}
}
fn format_unsupported_command(mode: Mode, settings: &Settings, text: &str) -> ParseNode {
let body: Vec<ParseNode> = text
.chars()
.map(|ch| ParseNode::TextOrd {
mode: Mode::Text,
loc: None,
text: ch.to_string(),
})
.collect();
ParseNode::Color {
mode,
color: settings.error_color.clone(),
body,
}
}
fn array_row_at_max(body: &[Vec<ParseNode>], max_columns: Option<usize>) -> bool {
max_columns.is_some_and(|maximum| {
body.last().is_some_and(|row| row.len() >= maximum)
})
}
fn unwrap_array_parse_result(
result: Result<ParseNode, ParseError>,
close_cell: Result<(), ParseError>,
close_array: Result<(), ParseError>,
) -> Result<ParseNode, ParseError> {
match (result, close_cell, close_array) {
(Err(err), _, _) => Err(err),
(_, Err(err), _) => Err(err),
(_, _, Err(err)) => Err(err),
(Ok(node), Ok(()), Ok(())) => Ok(node),
}
}
fn is_ascii_hex_digit(c: char) -> bool {
c.is_ascii_digit() || ('a'..='f').contains(&c) || ('A'..='F').contains(&c)
}
fn all_code_units(text: &str, predicate: impl Fn(char) -> bool) -> bool {
if text.is_empty() {
return false;
}
text.chars().all(predicate)
}
fn normalized_color(text: &str) -> Option<String> {
if let Some(digits) = text.strip_prefix('#') {
if (digits.len() == 3 || digits.len() == 4 || digits.len() == 6 || digits.len() == 8)
&& all_code_units(digits, is_ascii_hex_digit)
{
Some(text.to_string())
} else {
None
}
} else if text.chars().count() == 6 && all_code_units(text, is_ascii_hex_digit) {
Some(format!("#{text}"))
} else if all_code_units(text, is_ascii_alphabetic) {
Some(text.to_string())
} else {
None
}
}
fn is_url_escape_target(c: char) -> bool {
matches!(c, '#' | '$' | '%' | '&' | '~' | '_' | '^' | '{' | '}')
}
fn unescape_url(text: &str) -> String {
let chars: Vec<char> = text.chars().collect();
let mut builder = String::new();
let mut segment_start = 0;
let mut index = 0;
while index < chars.len() {
if chars[index] == '\\'
&& index + 1 < chars.len()
&& is_url_escape_target(chars[index + 1])
{
builder.extend(&chars[segment_start..index]);
builder.push(chars[index + 1]);
segment_start = index + 2;
index += 2;
continue;
}
index += 1;
}
builder.extend(&chars[segment_start..]);
builder
}