1use std::rc::Rc;
5
6use crate::Engine;
7use crate::cmd::Cmd;
8use crate::hash::{CsId, NO_CS};
9use crate::input::{Level, ScannerStatus, TokKind};
10use crate::primitives_gen::assign_int as ip;
11use crate::token::{Token, TokenList};
12
13pub const EXPAND_DEPTH: u32 = 10000;
15
16pub mod cs_name_codes {
18 pub const CSNAME: u32 = 0;
20 pub const LAST_NAMED_CS: u32 = 1;
22 pub const BEGIN_CSNAME: u32 = 2;
24}
25
26#[inline]
28fn is_match_or_end(t: Token) -> bool {
29 !t.is_cs() && (t.cmd_code() == Cmd::ParEnd as u32 || t.cmd_code() == Cmd::Stop as u32)
30}
31
32#[inline]
35fn is_brace(t: Token) -> bool {
36 !t.is_cs() && (t.cmd_code() == Cmd::LeftBrace as u32 || t.cmd_code() == Cmd::RightBrace as u32)
37}
38
39#[inline]
40fn is_left_brace(t: Token) -> bool {
41 !t.is_cs() && t.cmd_code() == Cmd::LeftBrace as u32
42}
43
44impl Engine {
45 pub fn get_x_token(&mut self) {
48 self.get_next();
49 if self.cur_cmd > Cmd::MAX_COMMAND {
50 self.get_x_token_expanding();
51 } else {
52 self.cur_tok = self.make_cur_tok();
53 }
54 }
55
56 #[inline(never)]
59 fn get_x_token_expanding(&mut self) {
60 loop {
61 if self.cur_cmd >= Cmd::Call {
62 if self.cur_cmd < Cmd::EndTemplate {
63 self.macro_call();
64 } else {
65 self.cur_cs = self.cs.frozen_endv;
66 self.cur_cmd = Cmd::Endv;
67 break;
68 }
69 } else {
70 self.expand();
71 }
72 self.get_next();
73 if self.cur_cmd <= Cmd::MAX_COMMAND {
74 break;
75 }
76 }
77 self.cur_tok = self.make_cur_tok();
78 }
79
80 pub fn x_token(&mut self) {
82 while self.cur_cmd > Cmd::MAX_COMMAND {
83 if self.cur_cmd == Cmd::EndTemplate {
84 self.cur_cs = self.cs.frozen_endv;
87 self.cur_cmd = Cmd::Endv;
88 break;
89 }
90 self.expand();
91 self.get_next();
92 }
93 self.cur_tok = self.make_cur_tok();
94 }
95
96 pub fn expand(&mut self) {
98 let cv = self.cur_val;
99 let cvl = self.cur_val_level;
100 let cg = self.cur_glue;
101 self.expand_depth += 1;
102 if self.expand_depth >= EXPAND_DEPTH {
103 self.overflow("expansion depth", i64::from(EXPAND_DEPTH));
104 }
105 loop {
106 if self.cur_cmd < Cmd::Call {
107 if self.eqtb.int(ip::TRACINGCOMMANDS) > 1 {
108 self.show_cur_cmd_chr();
109 }
110 match self.cur_cmd {
111 Cmd::TopBotMark => self.insert_mark_text(),
112 Cmd::ExpandAfter => {
113 if self.cur_chr == 0 {
114 self.get_token();
116 let t = self.cur_tok;
117 self.get_token();
118 if self.cur_cmd > Cmd::MAX_COMMAND {
119 self.expand();
120 } else {
121 self.back_input();
122 }
123 self.cur_tok = t;
124 self.back_input();
125 } else {
126 self.get_token();
128 if self.cur_cmd == Cmd::IfTest
129 && self.cur_chr != crate::cond::if_codes::IF_CASE
130 {
131 self.cur_chr += crate::cond::UNLESS_CODE;
132 continue; }
134 self.print_err("You can't use `");
135 self.print_esc_str("unless");
136 self.printer.print_str("' before `");
137 self.print_cmd_chr(self.cur_cmd, self.cur_chr);
138 self.printer.print_char('\'' as u32);
139 self.help(&["Continue, and I'll forget that it ever happened."]);
140 self.back_error();
141 }
142 }
143 Cmd::NoExpand => {
144 if self.cur_chr == 0 {
145 self.no_expand();
146 } else {
147 self.primitive_cmd();
148 if self.cur_cmd > Cmd::MAX_COMMAND && self.cur_cmd != Cmd::NoExpand {
149 continue;
150 }
151 if self.cur_cmd <= Cmd::MAX_COMMAND {
152 self.back_input();
153 }
154 }
155 }
156 Cmd::CsName => self.manufacture_cs_name(),
157 Cmd::Convert => self.conv_toks(),
158 Cmd::The => self.ins_the_toks(),
159 Cmd::IfTest => self.conditional(),
160 Cmd::FiOrElse => self.fi_or_else(),
161 Cmd::Input => self.input_command(),
162 Cmd::LuaExpandableCall | Cmd::LuaLocalCall => self.lua_expandable_call(),
163 Cmd::Variable | Cmd::Feedback => self.backend_expandable(),
164 _ => {
165 self.print_err("Undefined control sequence");
166 self.help(&[
167 "The control sequence at the end of the top line",
168 "of your error message was never \\def'ed. If you have",
169 "misspelled it (e.g., `\\hobx'), type `I' and the correct",
170 "spelling (e.g., `I\\hbox'). Otherwise just continue,",
171 "and I'll forget about whatever was undefined.",
172 ]);
173 self.error();
174 }
175 }
176 } else if self.cur_cmd < Cmd::EndTemplate {
177 self.macro_call();
178 } else {
179 self.cur_tok = Token::cs(self.cs.frozen_endv);
180 self.back_input();
181 }
182 break;
183 }
184 self.expand_depth -= 1;
185 self.cur_val = cv;
186 self.cur_val_level = cvl;
187 self.cur_glue = cg;
188 }
189
190 fn no_expand(&mut self) {
192 let save = self.input.scanner_status;
193 self.input.scanner_status = ScannerStatus::Normal;
194 self.get_token();
195 self.input.scanner_status = save;
196 let t = self.cur_tok;
197 if t.is_cs() {
198 while let Some(Level::Tokens(l)) = self.input.stack.last() {
200 if l.loc < l.list.len() || l.kind == TokKind::VTemplate {
201 break;
202 }
203 self.end_token_list();
204 }
205 let list: TokenList = Rc::from(vec![Token::cs(self.cs.frozen_dont_expand), t]);
206 self.begin_token_list(list, TokKind::BackedUp);
207 } else {
208 self.back_input();
209 }
210 }
211
212 fn primitive_cmd(&mut self) {
215 let save = self.input.scanner_status;
216 self.input.scanner_status = ScannerStatus::Normal;
217 self.get_token();
218 self.input.scanner_status = save;
219 if self.cur_cs != NO_CS
220 && let Some(p) = crate::primitives::lookup(self.hash.name(self.cur_cs))
221 {
222 self.cur_cmd = p.cmd;
223 self.cur_chr = p.chr;
224 let frozen = self.frozen_primitive(self.cur_cs);
226 self.cur_cs = frozen;
227 self.cur_tok = Token::cs(frozen);
228 if self.cur_cmd > Cmd::MAX_COMMAND {
229 return;
230 }
231 return;
232 }
233 self.cur_cmd = Cmd::Relax;
234 self.cur_chr = 0;
235 self.back_input();
236 self.cur_cmd = Cmd::NoExpand;
237 }
238
239 pub(crate) fn lookup_named_cs(&mut self, name: &[u8]) -> CsId {
243 let cs = if name.is_empty() {
244 NO_CS
245 } else {
246 match self.hash.lookup(name) {
247 Some(cs) if self.eqtb.meaning(cs).cmd != Cmd::UndefinedCs => cs,
248 _ => crate::hash::UNDEFINED_CS,
249 }
250 };
251 self.last_cs_name = cs;
252 cs
253 }
254
255 fn manufacture_cs_name(&mut self) {
257 if self.cur_chr == cs_name_codes::LAST_NAMED_CS {
258 if self.last_cs_name != NO_CS {
259 self.cur_tok = Token::cs(self.last_cs_name);
260 self.back_input();
261 }
262 return;
263 }
264 let begin = self.cur_chr == cs_name_codes::BEGIN_CSNAME;
265 let e = self.cond.is_in_csname;
266 self.cond.is_in_csname = true;
267 let mut name: Vec<u8> = Vec::new();
268 loop {
269 self.get_x_token();
270 if self.cur_cs != NO_CS {
271 break;
272 }
273 crate::utf8x::push(&mut name, self.cur_chr);
274 }
275 if self.cur_cmd != Cmd::EndCsName {
276 self.complain_missing_endcsname();
277 }
278 self.cond.is_in_csname = e;
279 if begin {
280 let cs = self.lookup_named_cs(&name);
281 if cs != NO_CS && cs != crate::hash::UNDEFINED_CS {
282 self.cur_tok = Token::cs(cs);
283 self.back_input();
284 }
285 return;
286 }
287 let cs = self.hash.lookup_or_insert(&name);
288 self.last_cs_name = cs;
289 if self.eqtb.meaning(cs).cmd == Cmd::UndefinedCs {
290 self.eq_define(
291 crate::eqtb::Loc::Cs(cs),
292 crate::eqtb::Val::Meaning(crate::eqtb::Meaning::prim(Cmd::Relax, RELAX_CHR)),
293 );
294 }
295 self.cur_tok = Token::cs(cs);
296 self.back_input();
297 }
298
299 pub fn complain_missing_endcsname(&mut self) {
301 self.print_err("Missing ");
302 self.print_esc_str("endcsname");
303 self.printer.print_str(" inserted");
304 self.help(&[
305 "The control sequence marked <to be read again> should",
306 "not appear between \\csname and \\endcsname.",
307 ]);
308 self.back_error();
309 }
310
311 pub fn insert_relax(&mut self) {
313 self.cur_tok = Token::cs(self.cur_cs);
314 self.back_input();
315 self.cur_tok = Token::cs(self.cs.frozen_relax);
316 self.back_input();
317 if let Some(Level::Tokens(t)) = self.input.stack.last_mut() {
318 t.kind = TokKind::Inserted;
319 }
320 }
321
322 fn input_command(&mut self) {
324 match self.cur_chr {
325 1 => self.input.force_eof = true,
326 2 | 3 => self.pseudo_start(self.cur_chr == 3),
327 _ => {
328 if self.files.name_in_progress {
329 self.insert_relax();
330 } else {
331 self.start_input();
332 }
333 }
334 }
335 }
336
337 fn insert_mark_text(&mut self) {
339 let mut code = self.cur_chr;
340 let class = if code >= 5 {
341 code -= 5;
342 self.scan_register_num();
343 self.cur_val as u32
344 } else {
345 0
346 };
347 if let Some(list) = self.marks.get(class, code)
348 && !list.is_empty()
349 {
350 self.begin_token_list(list, TokKind::MarkText);
351 }
352 }
353
354 pub fn macro_call(&mut self) {
356 self.check_snapshot();
357 if let Some(mac) = &self.eqtb.meaning(self.cur_cs).mac
359 && mac.tokens[0] == Token::END_MATCH
360 && self.eqtb.int(ip::TRACINGMACROS) <= 0
361 {
362 let tokens = mac.tokens.clone();
363 while let Some(Level::Tokens(t)) = self.input.stack.last() {
364 if t.loc < t.list.len() || t.kind == TokKind::VTemplate {
365 break;
366 }
367 self.end_token_list();
368 }
369 let param_start = self.input.param_stack.len();
370 self.begin_macro(tokens, 1, self.cur_cs, param_start);
371 return;
372 }
373 let save_scanner_status = self.input.scanner_status;
374 let save_warning_index = self.input.warning_index;
375 self.input.warning_index = self.cur_cs;
376 let Some(mac) = self.eqtb.meaning(self.cur_cs).mac.clone() else {
377 self.input.scanner_status = save_scanner_status;
378 self.input.warning_index = save_warning_index;
379 return;
380 };
381 let tokens = mac.tokens.clone();
382 if self.eqtb.int(ip::TRACINGMACROS) > 0 {
383 self.begin_diagnostic();
384 self.printer.print_ln();
385 self.print_cs(self.input.warning_index);
386 self.token_show_list(&tokens, 0, tokens.len(), 10_000_000);
387 self.end_diagnostic(false);
388 }
389 let mut r = 0usize;
390 let mut pstack: Vec<TokenList> = std::mem::take(&mut self.param_scratch);
392 pstack.clear();
393 if tokens[0] != Token::END_MATCH {
394 self.input.scanner_status = ScannerStatus::Matching;
395 let mut unbalance: i32 = 0;
396 self.long_state = self.cur_cmd;
397 if self.long_state >= Cmd::OuterCall {
398 self.long_state = if self.long_state == Cmd::OuterCall {
399 Cmd::Call
400 } else {
401 Cmd::LongCall
402 };
403 }
404 let par_token = Token::cs(self.cs.par);
405 loop {
406 let mut p: Vec<Token> = std::mem::take(&mut self.arg_scratch);
409 p.clear();
410 let s: Option<usize>;
411 let mut match_chr = 0u32;
412 let mut m = 0i32;
413 let mut rbrace_ptr = 0usize;
414 if !is_match_or_end(tokens[r]) || tokens[r] == Token::END_MATCH {
415 s = None;
416 } else {
417 match_chr = tokens[r].chr();
418 s = Some(r + 1);
419 r += 1;
420 }
421 'cont: loop {
422 self.get_token();
423 if self.cur_tok == tokens[r] {
424 r += 1;
425 if is_match_or_end(tokens[r]) {
426 if is_left_brace(self.cur_tok) {
427 self.input.align_state -= 1;
428 }
429 break 'cont; }
431 continue 'cont;
432 }
433 if s != Some(r) {
435 match s {
436 None => {
437 self.print_err("Use of ");
438 self.sprint_cs(self.input.warning_index);
439 self.printer.print_str(" doesn't match its definition");
440 self.help(&[
441 "If you say, e.g., `\\def\\a1{...}', then you must always",
442 "put `1' after `\\a', since control sequence names are",
443 "made up of letters only. The macro here has not been",
444 "followed by the required stuff, so I'm ignoring it.",
445 ]);
446 self.error();
447 self.input.scanner_status = save_scanner_status;
448 self.input.warning_index = save_warning_index;
449 return;
450 }
451 Some(s) => {
452 let mut t = s;
453 loop {
454 p.push(tokens[t]);
455 m += 1;
456 let mut u = t + 1;
457 let mut v = s;
458 let mut resumed = false;
459 loop {
460 if u == r {
461 if self.cur_tok != tokens[v] {
462 break;
463 }
464 r = v + 1;
465 resumed = true;
466 break;
467 }
468 if tokens[u] != tokens[v] {
469 break;
470 }
471 u += 1;
472 v += 1;
473 }
474 if resumed {
475 self.runaway_list = p;
476 p = std::mem::take(&mut self.runaway_list);
477 continue 'cont;
478 }
479 t += 1;
480 if t == r {
481 break;
482 }
483 }
484 r = s;
485 }
486 }
487 }
488 if self.cur_tok == par_token && self.long_state != Cmd::LongCall {
489 self.runaway_list = p;
490 self.report_runaway_argument();
491 self.input.align_state -= unbalance;
492 self.input.scanner_status = save_scanner_status;
493 self.input.warning_index = save_warning_index;
494 return;
495 }
496 if is_brace(self.cur_tok) {
497 if is_left_brace(self.cur_tok) {
498 unbalance = 1;
500 loop {
501 p.push(self.cur_tok);
502 self.get_token();
503 if self.cur_tok == par_token && self.long_state != Cmd::LongCall {
504 self.runaway_list = p;
505 self.report_runaway_argument();
506 self.input.align_state -= unbalance;
507 self.input.scanner_status = save_scanner_status;
508 self.input.warning_index = save_warning_index;
509 return;
510 }
511 if is_brace(self.cur_tok) {
512 if is_left_brace(self.cur_tok) {
513 unbalance += 1;
514 } else {
515 unbalance -= 1;
516 if unbalance == 0 {
517 break;
518 }
519 }
520 }
521 }
522 rbrace_ptr = p.len();
523 p.push(self.cur_tok);
524 } else {
525 self.back_input();
527 self.print_err("Argument of ");
528 self.sprint_cs(self.input.warning_index);
529 self.printer.print_str(" has an extra }");
530 self.help(&[
531 "I've run across a `}' that doesn't seem to match anything.",
532 "For example, `\\def\\a#1{...}' and `\\a}' would produce",
533 "this error. If you simply proceed now, the `\\par' that",
534 "I've just inserted will cause me to report a runaway",
535 "argument that might be the root of the problem. But if",
536 "your `}' was spurious, just type `2' and it will go away.",
537 ]);
538 self.input.align_state += 1;
539 self.long_state = Cmd::Call;
540 self.cur_tok = par_token;
541 self.ins_error();
542 continue 'cont;
543 }
544 } else {
545 if self.cur_tok == Token::SPACE && is_match_or_end(tokens[r]) {
546 continue 'cont;
547 }
548 p.push(self.cur_tok);
549 }
550 m += 1;
551 if !is_match_or_end(tokens[r]) {
552 continue 'cont;
553 }
554 break 'cont;
555 }
556 if s.is_some() {
558 let param: TokenList =
560 if m == 1 && p.last().is_some_and(|&t| is_brace(t)) && rbrace_ptr > 0 {
561 Rc::from(&p[1..rbrace_ptr])
562 } else {
563 Rc::from(&p[..])
564 };
565 pstack.push(param);
566 let param = &pstack[pstack.len() - 1];
567 if self.eqtb.int(ip::TRACINGMACROS) > 0 {
568 self.begin_diagnostic();
569 self.printer.begin_line();
570 self.printer.print_char(match_chr);
571 self.printer.print_int(pstack.len() as i64);
572 self.printer.print_str("<-");
573 self.show_token_list(param, 0, param.len(), 1000);
574 self.end_diagnostic(false);
575 }
576 }
577 self.arg_scratch = p;
578 if tokens[r] == Token::END_MATCH {
579 break;
580 }
581 }
582 }
583 while let Some(Level::Tokens(t)) = self.input.stack.last() {
585 if t.loc < t.list.len() || t.kind == TokKind::VTemplate {
586 break;
587 }
588 self.end_token_list();
589 }
590 let param_start = self.input.param_stack.len();
591 self.input.param_stack.append(&mut pstack);
592 self.param_scratch = pstack;
593 self.begin_macro(tokens, r + 1, self.input.warning_index, param_start);
594 self.input.scanner_status = save_scanner_status;
595 self.input.warning_index = save_warning_index;
596 }
597
598 fn report_runaway_argument(&mut self) {
600 if self.long_state == Cmd::Call {
601 self.runaway();
602 self.print_err("Paragraph ended before ");
603 self.sprint_cs(self.input.warning_index);
604 self.printer.print_str(" was complete");
605 self.help(&[
606 "I suspect you've forgotten a `}', causing me to apply this",
607 "control sequence to too much text. How can we recover?",
608 "My plan is to forget the whole thing and hope for the best.",
609 ]);
610 self.back_error();
611 }
612 }
613
614 pub fn runaway(&mut self) {
616 let what = match self.input.scanner_status {
617 ScannerStatus::Defining => "definition",
618 ScannerStatus::Matching => "argument",
619 ScannerStatus::Aligning => "preamble",
620 ScannerStatus::Absorbing => "text",
621 _ => return,
622 };
623 self.printer.print_nl("Runaway ");
624 self.printer.print_str(what);
625 self.printer.print_char('?' as u32);
626 self.printer.print_ln();
627 let list = std::mem::take(&mut self.runaway_list);
628 let limit = self.printer.error_line.saturating_sub(10);
629 self.show_token_list(&list, 0, list.len(), limit as i64);
630 self.runaway_list = list;
631 }
632
633 pub fn check_outer_validity(&mut self) {
636 if self.input.scanner_status == ScannerStatus::Normal {
637 return;
638 }
639 if self.eqtb.int(ip::SUPPRESSOUTERERROR) > 0 && self.cur_cs != NO_CS {
640 return;
641 }
642 self.err.deletions_allowed = false;
643 if self.cur_cs != NO_CS {
645 let reread = match self.input.stack.last() {
646 Some(Level::Tokens(_)) => true,
647 Some(Level::Text(t)) => !matches!(
648 t.source,
649 crate::input::LineSource::Terminal | crate::input::LineSource::Read(_)
650 ),
651 None => true,
652 };
653 if reread {
654 let list: TokenList = Rc::from(vec![Token::cs(self.cur_cs)]);
655 self.back_list(list);
656 }
657 self.cur_cmd = Cmd::Spacer;
658 self.cur_chr = ' ' as u32;
659 }
660 if self.input.scanner_status > ScannerStatus::Skipping {
661 self.runaway();
662 if self.cur_cs == NO_CS {
663 self.print_err("File ended");
664 } else {
665 self.cur_cs = NO_CS;
666 self.print_err("Forbidden control sequence found");
667 }
668 self.printer.print_str(" while scanning ");
669 let mut ins: Vec<Token> = Vec::new();
670 match self.input.scanner_status {
671 ScannerStatus::Defining => {
672 self.printer.print_str("definition");
673 ins.push(Token::RIGHT_BRACE);
674 }
675 ScannerStatus::Matching => {
676 self.printer.print_str("use");
677 ins.push(Token::cs(self.cs.par));
678 self.long_state = Cmd::OuterCall;
679 }
680 ScannerStatus::Aligning => {
681 self.printer.print_str("preamble");
682 ins.push(Token::cs(self.cs.frozen_cr));
683 ins.push(Token::RIGHT_BRACE);
684 self.input.align_state = -1_000_000;
685 }
686 _ => {
687 self.printer.print_str("text");
688 ins.push(Token::RIGHT_BRACE);
689 }
690 }
691 self.ins_list(Rc::from(ins));
692 self.printer.print_str(" of ");
693 self.sprint_cs(self.input.warning_index);
694 self.help(&[
695 "I suspect you have forgotten a `}', causing me",
696 "to read past where you wanted me to stop.",
697 "I'll try to recover; but if the error is serious,",
698 "you'd better type `E' or `X' now and fix your file.",
699 ]);
700 self.error();
701 } else {
702 self.print_err("Incomplete ");
703 self.print_cmd_chr(Cmd::IfTest, self.cond.cur_if);
704 self.printer.print_str("; all text was ignored after line ");
705 self.printer.print_int(self.cond.skip_line.into());
706 let third = if self.cur_cs != NO_CS {
707 self.cur_cs = NO_CS;
708 "the matching `\\fi'. I've inserted a `\\fi'; this might work."
709 } else {
710 "The file ended while I was skipping conditional text."
711 };
712 self.help(&[
713 "A forbidden control sequence occurred in skipped text.",
714 "This kind of error happens when you say `\\if...' and forget",
715 third,
716 ]);
717 self.cur_tok = Token::cs(self.cs.frozen_fi);
718 self.ins_error();
719 }
720 self.err.deletions_allowed = true;
721 }
722
723 fn frozen_primitive(&mut self, cs: CsId) -> CsId {
726 let name = self.hash.name(cs).to_vec();
727 if let Some(&f) = self.frozen_prims.get(&name) {
728 return f;
729 }
730 let f = self.hash.new_frozen(&name);
731 if let Some(p) = crate::primitives::lookup(&name) {
732 self.eqtb.ensure_cs(f);
733 self.eqtb.cs[f.0 as usize].val = crate::eqtb::Meaning::prim(p.cmd, p.chr);
734 }
735 self.frozen_prims.insert(name, f);
736 f
737 }
738}
739
740pub const RELAX_CHR: u32 = 0x11_0000;