1pub(crate) mod json;
31pub(crate) mod limits;
32pub(crate) mod math;
33pub(crate) mod number;
34pub(crate) mod source;
35pub(crate) mod strings;
36pub(crate) mod typst_dict;
37pub(crate) mod yaml;
38
39use crate::canonicalize::canonicalize_document;
40use crate::diagnostic::{DiagnosticError, codes};
41
42#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
44pub enum DocumentFormat {
45 Markdown,
47 Typst,
49}
50
51fn printable(doc: &serde_json::Map<String, serde_json::Value>) -> json::Json {
54 let canon = canonicalize_document(&serde_json::Value::Object(doc.clone()));
55 json::sort_keys(&json::from_serde(&canon))
56}
57
58pub fn print_markdown(doc: &serde_json::Map<String, serde_json::Value>) -> String {
64 source::print_md(&printable(doc))
65}
66
67pub fn print_typst(doc: &serde_json::Map<String, serde_json::Value>) -> String {
70 source::print_typ(&printable(doc))
71}
72
73pub fn print_document(
75 doc: &serde_json::Map<String, serde_json::Value>,
76 format: DocumentFormat,
77) -> String {
78 match format {
79 DocumentFormat::Markdown => print_markdown(doc),
80 DocumentFormat::Typst => print_typst(doc),
81 }
82}
83
84pub fn print_math(expr: &serde_json::Value, names: &[String], format: DocumentFormat) -> String {
98 let e = json::sort_keys(&json::from_serde(&canonicalize_document(expr)));
99 let sides = match &e {
100 json::Json::Obj(kvs) => math::expr::look("lhs", kvs).zip(math::expr::look("rhs", kvs)),
101 _ => None,
102 };
103 let e = match sides {
104 Some((lhs, rhs)) => json::Json::Obj(vec![
105 (
106 "args".to_string(),
107 json::Json::Arr(vec![lhs.clone(), rhs.clone()]),
108 ),
109 ("op".to_string(), json::Json::Str("==".to_string())),
110 ]),
111 None => e,
112 };
113 let sym = math::Sym {
114 names: names.to_vec(),
115 templates: Vec::new(),
116 };
117 match format {
118 DocumentFormat::Markdown => math::print_latex(&sym, &e),
119 DocumentFormat::Typst => math::print_typst(&sym, &e),
120 }
121}
122
123pub fn expression_names(expr: &serde_json::Value) -> Vec<String> {
128 fn walk(v: &serde_json::Value, out: &mut Vec<String>) {
129 match v {
130 serde_json::Value::String(name) if !out.contains(name) => out.push(name.clone()),
131 serde_json::Value::Array(items) => items.iter().for_each(|i| walk(i, out)),
132 serde_json::Value::Object(fields) => {
133 for key in ["lhs", "rhs", "args"] {
134 if let Some(child) = fields.get(key) {
135 walk(child, out);
136 }
137 }
138 }
139 _ => {}
140 }
141 }
142 let mut out = Vec::new();
143 walk(expr, &mut out);
144 out
145}
146
147#[derive(Clone, Debug, PartialEq, Eq)]
149pub struct MathError {
150 pub code: &'static str,
152 pub message: String,
154 pub at: Option<usize>,
161}
162
163impl std::fmt::Display for MathError {
164 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
165 write!(f, "[{}] {}", self.code, self.message)?;
166 if let Some(at) = self.at {
167 write!(f, " (at character {at})")?;
168 }
169 Ok(())
170 }
171}
172
173impl std::error::Error for MathError {}
174
175pub fn parse_math(
197 text: &str,
198 names: &[String],
199 format: DocumentFormat,
200) -> Result<serde_json::Value, MathError> {
201 let sym = math::Sym {
202 names: names.to_vec(),
203 templates: Vec::new(),
204 };
205 let cs: Vec<char> = text.chars().collect();
206 let (read, found) = limits::reading(|| {
207 let (sf, lexed) = match format {
208 DocumentFormat::Markdown => (math::token::Sf::Latex, math::latex::lex_at(&cs)),
209 DocumentFormat::Typst => (math::token::Sf::Typst, math::typst::lex_at(&cs)),
210 };
211 let (toks, starts) = lexed?;
212 limits::track(&toks);
213 Ok((math::parse_toks(sf, &sym, &toks), starts))
214 });
215 let (e, at) = match read {
216 Ok((e, starts)) => {
217 let at = found
218 .reach
219 .map(|i| starts.get(i).copied().unwrap_or(cs.len()));
220 (e, at)
221 }
222 Err(at) => (None, Some(at)),
223 };
224 let e = e.map(|e| match source::equation::split_eq(e) {
225 Ok((lhs, rhs)) => json::Json::Obj(vec![("lhs".to_string(), lhs), ("rhs".to_string(), rhs)]),
226 Err(e) => e,
227 });
228 if e.is_none() && found.removed_op.is_none() && !found.too_deep {
229 return Err(MathError {
230 code: codes::PARSE_ERROR,
231 message: match format {
232 DocumentFormat::Markdown => "not an ESM expression in LaTeX math".to_string(),
233 DocumentFormat::Typst => "not an ESM expression in Typst math".to_string(),
234 },
235 at,
236 });
237 }
238 checked(e, found, format).map_err(|d| MathError {
239 code: d.code,
240 message: d.message,
241 at: None,
242 })
243}
244
245pub fn parse_markdown(text: &str) -> Result<serde_json::Value, DiagnosticError> {
256 read(text, DocumentFormat::Markdown)
257}
258
259pub fn parse_typst(text: &str) -> Result<serde_json::Value, DiagnosticError> {
265 read(text, DocumentFormat::Typst)
266}
267
268pub fn parse_document(
274 text: &str,
275 format: DocumentFormat,
276) -> Result<serde_json::Value, DiagnosticError> {
277 read(text, format)
278}
279
280#[derive(Clone, Debug, PartialEq, Eq)]
283pub struct ConstructSource {
284 pub pointer: String,
290 pub span: std::ops::Range<usize>,
292}
293
294pub fn parse_document_with_sources(
306 text: &str,
307 format: DocumentFormat,
308) -> Result<(serde_json::Value, Vec<ConstructSource>), DiagnosticError> {
309 let (read, found) = limits::reading(|| match format {
310 DocumentFormat::Markdown => source::parse_md_sourced(text),
311 DocumentFormat::Typst => source::parse_typ_sourced(text),
312 });
313 let (doc, spans) = match read {
314 Some((d, s)) => (Some(d), s),
315 None => (None, Vec::new()),
316 };
317 let doc = checked(doc, found, format)?;
318 let spans = spans
319 .into_iter()
320 .map(|(pointer, span)| ConstructSource { pointer, span })
321 .collect();
322 Ok((doc, spans))
323}
324
325fn read(text: &str, format: DocumentFormat) -> Result<serde_json::Value, DiagnosticError> {
326 let (doc, found) = limits::reading(|| match format {
327 DocumentFormat::Markdown => source::parse_md(text),
328 DocumentFormat::Typst => source::parse_typ(text),
329 });
330 checked(doc, found, format)
331}
332
333fn checked(
335 doc: Option<json::Json>,
336 found: limits::Findings,
337 format: DocumentFormat,
338) -> Result<serde_json::Value, DiagnosticError> {
339 let too_deep = found.too_deep
340 || doc
341 .as_ref()
342 .is_some_and(|d| limits::json_depth(d) > limits::MAX_JSON_DEPTH);
343 if too_deep {
344 return Err(DiagnosticError {
345 code: codes::PARSE_ERROR,
346 message: format!(
347 "the document nests too deeply for the reader (at most {} constructs open at once, and {} levels of arrays and objects)",
348 limits::MAX_NESTING,
349 limits::MAX_JSON_DEPTH
350 ),
351 });
352 }
353 match doc {
354 Some(d) => json::to_serde(&d).map_err(|n| DiagnosticError {
355 code: codes::PARSE_ERROR,
356 message: format!(
357 "the number `{}` is outside the binary64 range",
358 number::print(&n).into_iter().collect::<String>()
359 ),
360 }),
361 None => Err(refusal(found.removed_op, format)),
362 }
363}
364
365fn refusal(removed_op: Option<&str>, format: DocumentFormat) -> DiagnosticError {
370 match removed_op {
371 Some(op) => DiagnosticError {
372 code: codes::REMOVED_OP,
373 message: format!("`{op}` was removed from the format (esm-spec §4.2)"),
374 },
375 None => DiagnosticError {
376 code: codes::PARSE_ERROR,
377 message: match format {
378 DocumentFormat::Markdown => "not an ESM Markdown document".to_string(),
379 DocumentFormat::Typst => "not an ESM Typst document".to_string(),
380 },
381 },
382 }
383}
384
385#[cfg(test)]
386mod tests {
387 use super::*;
388
389 const MD_HEAD: &str = "---\nesm: \"2.3.0\"\nname: \"fixture\"\n---\n\n::model[M]\n\n:param[$k$]{}\n\n:var[$y$]{}\n\n";
390 const TYP_HEAD: &str = "#import \"@preview/esm:0.1.0\": *\n#show: esm-document.with(esm: \"2.3.0\", name: \"fixture\")\n\n#model(\"M\")\n\n#param($k$)\n\n#var($y$)\n\n";
391
392 fn md(body: &str) -> String {
393 format!("{MD_HEAD}{body}")
394 }
395
396 fn typ(body: &str) -> String {
397 format!("{TYP_HEAD}{body}")
398 }
399
400 fn code(r: Result<serde_json::Value, DiagnosticError>) -> (&'static str, String) {
401 let e = r.expect_err("refused");
402 (e.code, e.message)
403 }
404
405 fn math_code(r: Result<serde_json::Value, MathError>) -> (&'static str, String) {
406 let e = r.expect_err("refused");
407 (e.code, e.message)
408 }
409
410 #[test]
411 fn parse_math_reads_an_expression_or_an_equation() {
412 let names = vec!["RT".to_string(), "k".to_string(), "y".to_string()];
413 let e = parse_math("RT + k", &names, DocumentFormat::Markdown).unwrap();
414 assert_eq!(e, serde_json::json!({"op": "+", "args": ["RT", "k"]}));
415 let rt = vec!["R".to_string(), "T".to_string()];
418 let e = parse_math("RT", &rt, DocumentFormat::Markdown).unwrap();
419 assert_eq!(e, serde_json::json!({"op": "*", "args": ["R", "T"]}));
420 assert!(parse_math("RT", &[], DocumentFormat::Markdown).is_err());
421 let e = parse_math("y = 2 k", &names, DocumentFormat::Typst).unwrap();
422 assert_eq!(
423 e,
424 serde_json::json!({"lhs": "y", "rhs": {"op": "*", "args": [2, "k"]}})
425 );
426 for (text, format) in [
427 ("y = k", DocumentFormat::Markdown),
428 ("(dif y)/(dif t) = -k y", DocumentFormat::Typst),
429 ] {
430 let e = parse_math(text, &names, format).unwrap();
431 assert_eq!(print_math(&e, &names, format), text);
432 }
433 }
434
435 #[test]
436 fn parse_math_refuses_as_a_document_read_does() {
437 let names = vec!["k".to_string()];
438 let (c, _) = math_code(parse_math("k +", &names, DocumentFormat::Markdown));
439 assert_eq!(c, codes::PARSE_ERROR);
440 let (c, _) = math_code(parse_math("1.0e400 k", &names, DocumentFormat::Typst));
441 assert_eq!(c, codes::PARSE_ERROR);
442 let (c, m) = math_code(parse_math(
443 "\\operatorname{arrayop}(k)",
444 &names,
445 DocumentFormat::Markdown,
446 ));
447 assert_eq!(c, codes::REMOVED_OP, "{m}");
448 let deep = format!("{}k{}", "(".repeat(50_000), ")".repeat(50_000));
449 let (c, _) = math_code(parse_math(&deep, &names, DocumentFormat::Typst));
450 assert_eq!(c, codes::PARSE_ERROR);
451 }
452
453 #[test]
454 fn parse_math_says_where_the_math_goes_wrong() {
455 let names = vec!["k".to_string(), "y".to_string()];
456 let at = |text: &str, format| parse_math(text, &names, format).unwrap_err().at;
457 assert_eq!(at("y = k ? 2", DocumentFormat::Typst), Some(6));
459 assert_eq!(at("y = k \\@", DocumentFormat::Markdown), Some(6));
460 assert_eq!(at("λ = k", DocumentFormat::Typst), Some(0));
461 assert_eq!(at("y = k ) + 1", DocumentFormat::Typst), Some(6));
463 assert_eq!(at("y = k ) + 1", DocumentFormat::Markdown), Some(6));
464 assert_eq!(at("y = -k +", DocumentFormat::Typst), Some(8));
466 assert_eq!(at("y = -k +", DocumentFormat::Markdown), Some(8));
467 assert_eq!(at("y_k = k )", DocumentFormat::Typst), Some(8));
469 let e = parse_math(
471 "\\operatorname{arrayop}(k)",
472 &names,
473 DocumentFormat::Markdown,
474 );
475 assert_eq!(e.unwrap_err().at, None);
476 }
477
478 #[test]
479 fn a_number_beyond_binary64_is_refused_not_zeroed() {
480 let (c, m) = code(parse_markdown(&md("$$\ny = 1.0e400 k\n$$\n")));
481 assert_eq!(c, codes::PARSE_ERROR);
482 assert!(m.contains("1.0e400") && m.contains("binary64"), "{m}");
483 let (c, _) = code(parse_typst(&typ("$ y = 1.0e400 k $\n")));
484 assert_eq!(c, codes::PARSE_ERROR);
485 let (c, _) = code(parse_markdown(&md("$$\ny = 1.0e4294967296 k\n$$\n")));
487 assert_eq!(c, codes::PARSE_ERROR);
488 let d = parse_markdown(&md("$$\ny = 1.0e-400 k\n$$\n")).unwrap();
490 assert_eq!(
491 d["models"]["M"]["equations"][0]["rhs"]["args"][0],
492 serde_json::json!(0)
493 );
494 }
495
496 #[test]
497 fn deep_nesting_is_refused_without_exhausting_the_stack() {
498 let n = 50_000;
499 let deep = |open: &str, close: &str| {
500 format!("$$\ny = {}k{}\n$$\n", open.repeat(n), close.repeat(n))
501 };
502 for text in [
503 md(&deep("(", ")")),
504 md(&deep("\\frac{", "}{k}")),
505 md(&deep("-", "")),
506 md(&deep("\\left(", "\\right)")),
507 typ(&format!("$ y = {}k{} $\n", "(".repeat(n), ")".repeat(n))),
508 ] {
509 let format = if text.starts_with("---") {
510 DocumentFormat::Markdown
511 } else {
512 DocumentFormat::Typst
513 };
514 let (c, m) = code(parse_document(&text, format));
515 assert_eq!(c, codes::PARSE_ERROR);
516 assert!(m.contains("nests too deeply"), "{m}");
517 }
518 let chain = format!("$$\ny = k{}\n$$\n", " - k".repeat(n));
520 let (_, m) = code(parse_markdown(&md(&chain)));
521 assert!(m.contains("nests too deeply"), "{m}");
522 let mut yaml = String::from("```esm:metaparameters\nx:\n");
524 for i in 1..2000 {
525 yaml.push_str(&" ".repeat(2 * i));
526 yaml.push_str("-\n");
527 }
528 yaml.push_str("```\n");
529 let (_, m) = code(parse_markdown(&md(&yaml)));
530 assert!(m.contains("nests too deeply"), "{m}");
531 let flow = format!(
532 ":var[$z$]{{default='{}1{}'}}\n",
533 "[".repeat(n),
534 "]".repeat(n)
535 );
536 let (_, m) = code(parse_markdown(&md(&flow)));
537 assert!(m.contains("nests too deeply"), "{m}");
538 let ok = format!("$$\ny = {}k{}\n$$\n", "(".repeat(20), ")".repeat(20));
540 assert!(parse_markdown(&md(&ok)).is_ok());
541 }
542
543 #[test]
544 fn the_deepest_nesting_read_fits_a_wasm_sized_stack() {
545 let deepest = |open: &str, close: &str| {
549 (1..)
550 .map(|n| {
551 md(&format!(
552 "$$\ny = {}k{}\n$$\n",
553 open.repeat(n),
554 close.repeat(n)
555 ))
556 })
557 .take_while(|t| parse_markdown(t).is_ok())
558 .last()
559 .expect("one level reads")
560 };
561 let texts = [deepest("(", ")"), deepest("\\frac{", "}{k}")];
562 std::thread::Builder::new()
563 .stack_size(1 << 20)
564 .spawn(move || {
565 for t in &texts {
566 parse_markdown(t).unwrap();
567 print_markdown(parse_markdown(t).unwrap().as_object().unwrap());
568 }
569 })
570 .unwrap()
571 .join()
572 .expect("read within 1 MiB of stack");
573 }
574
575 #[test]
576 fn a_removed_operator_counts_only_in_math() {
577 let removed = "$$\ny = \\operatorname{arrayop}(k)\n$$\n";
578 assert_eq!(code(parse_markdown(&md(removed))).0, codes::REMOVED_OP);
579 for prose in [
581 "The old `\\operatorname{arrayop}` is gone.\n\n",
582 "```\n\\esm{arrayop}{}{k}\n```\n\n",
583 "Write \\operatorname{fn} no more.\n\n",
584 ] {
585 let text = md(&format!("{prose}$$\ny = k +\n$$\n"));
586 assert_eq!(code(parse_markdown(&text)).0, codes::PARSE_ERROR, "{prose}");
587 }
588 let typ_removed = typ("$ y = op(\"fn\")(k) $\n");
589 assert_eq!(code(parse_typst(&typ_removed)).0, codes::REMOVED_OP);
590 for prose in ["Call stop(\"fn\") here.\n\n", "No more op(\"rank\").\n\n"] {
592 let text = typ(&format!("{prose}$ y = k + $\n"));
593 assert_eq!(code(parse_typst(&text)).0, codes::PARSE_ERROR, "{prose}");
594 }
595 }
596
597 #[test]
598 fn sources_point_at_what_each_construct_reads_as() {
599 let text = md(concat!(
600 "$$\n\\begin{aligned}y &= k \\\\\ny &= 2 k\\end{aligned}\n$$\n\n",
601 "$$\ny = 3 k\n$$\n\n",
602 "$$\n\\operatorname{loss}(r, v) \\coloneqq -r v\n$$\n",
603 ));
604 let (doc, found) = parse_document_with_sources(&text, DocumentFormat::Markdown).unwrap();
605 assert_eq!(doc, parse_markdown(&text).unwrap());
606 let at = |p: &str| {
607 let s = found
608 .iter()
609 .find(|s| s.pointer == p)
610 .unwrap_or_else(|| panic!("{p}"));
611 &text[s.span.clone()]
612 };
613 assert!(at("/models/M").starts_with("::model[M]"));
614 assert_eq!(at("/models/M/variables/k"), ":param[$k$]{}");
615 assert!(at("/models/M/equations/0").contains("aligned"));
616 assert_eq!(at("/models/M/equations/0"), at("/models/M/equations/1"));
617 assert!(at("/models/M/equations/2").contains("3 k"));
618 assert!(at("/models/M/expression_templates/loss").contains("coloneqq"));
619 }
620}