badness_parser/semantic/expl3/
calls.rs1use std::collections::{HashMap, VecDeque};
10use std::sync::Arc;
11
12use rowan::{TextRange, TextSize};
13
14use crate::ast::{AstNode, Command, Environment, Group, command_name};
15use crate::parser::lexer::{ExplToggle, expl_toggle};
16use crate::semantic::expl3::{Expl3Slot, expl3_slots};
17use crate::syntax::{SyntaxElement, SyntaxKind, SyntaxNode, is_trivia};
18
19#[derive(Default)]
20pub struct Expl3Index {
21 calls: HashMap<TextSize, Call>,
22}
23
24pub struct Call {
25 pub name: String,
26 pub arguments: Vec<Argument>,
27 pub message_parameters: Vec<(TextRange, u8)>,
29}
30
31pub struct Argument {
32 pub specifier: u8,
33 elements: Vec<SyntaxElement>,
34}
35
36pub struct Literal {
37 pub text: String,
38 pub range: TextRange,
39}
40
41struct Stream {
42 elements: VecDeque<SyntaxElement>,
43 definitions: Arc<[u8]>,
44 active: bool,
45}
46
47impl Expl3Index {
48 pub fn calls(&self) -> impl Iterator<Item = &Call> {
50 let mut calls: Vec<_> = self.calls.iter().collect();
51 calls.sort_unstable_by_key(|(offset, _)| **offset);
52 calls.into_iter().map(|(_, call)| call)
53 }
54
55 pub fn get(&self, node: &SyntaxNode) -> Option<&Call> {
56 self.calls.get(&node.text_range().start())
57 }
58
59 pub fn build(root: &SyntaxNode) -> Self {
60 let mut index = Self::default();
61 let documentation = root
62 .descendants_with_tokens()
63 .any(|el| el.kind() == SyntaxKind::DOC_MARGIN);
64 let mut pending = if documentation {
67 root.descendants()
68 .filter_map(Environment::cast)
69 .filter(|env| {
70 env.name()
71 .is_some_and(|name| matches!(name.as_str(), "macrocode" | "macrocode*"))
72 })
73 .map(|env| environment_stream(env.syntax(), Arc::from([]), true))
74 .collect()
75 } else {
76 vec![Stream {
77 elements: root.children_with_tokens().collect(),
78 definitions: Arc::from([]),
79 active: false,
80 }]
81 };
82 while let Some(mut stream) = pending.pop() {
83 let mut known_boundary = true;
84 while let Some(element) = stream.elements.pop_front() {
85 let Some(node) = element.as_node() else {
86 if stream.active && !is_stream_trivia(element.kind()) {
89 known_boundary = false;
90 }
91 continue;
92 };
93 match node.kind() {
94 SyntaxKind::PARAGRAPH | SyntaxKind::TEXT => {
95 for child in node
96 .children_with_tokens()
97 .collect::<Vec<_>>()
98 .into_iter()
99 .rev()
100 {
101 stream.elements.push_front(child);
102 }
103 }
104 SyntaxKind::DOC_COMMENT => {}
105 SyntaxKind::ENVIRONMENT => {
106 let Some(env) = Environment::cast(node.clone()) else {
107 continue;
108 };
109 let Some(name) = env.name() else { continue };
110 if known_boundary
111 && matches!(name.as_str(), "document" | "macrocode" | "macrocode*")
112 {
113 pending.push(environment_stream(
114 node,
115 stream.definitions.clone(),
116 stream.active || name.starts_with("macrocode"),
117 ));
118 }
119 }
120 SyntaxKind::GROUP if known_boundary => {
121 if let Some(group) = Group::cast(node.clone()).filter(closed_group) {
122 pending.push(body_stream(
123 &group,
124 stream.definitions.clone(),
125 stream.active,
126 ));
127 }
128 }
129 SyntaxKind::COMMAND => {
130 let Some(name) = command_name(node) else {
131 known_boundary &= !stream.active;
132 continue;
133 };
134 if let Some(toggle) = expl_toggle(&format!("\\{name}")) {
135 if (known_boundary || !stream.active) && !toggle_may_be_an_operand(node)
136 {
137 stream.active = toggle == ExplToggle::On;
138 known_boundary = true;
139 } else {
140 stream.active = true;
143 known_boundary = false;
144 }
145 continue;
146 }
147 if !known_boundary {
148 continue;
149 }
150 let Some(mut call) = read_call(node) else {
151 known_boundary &= !stream.active;
152 continue;
153 };
154 stream.active = true;
155 if call.name.starts_with("exp_") || call.name.starts_with("use:") {
156 known_boundary = false;
159 continue;
160 }
161 if let Some((body, count)) = definition_body(&call, &stream.definitions) {
162 let mut definitions = stream.definitions.to_vec();
163 definitions.push(count);
164 pending.push(body_stream(&body, definitions.into(), true));
165 }
166 for argument in &call.arguments {
167 if matches!(argument.specifier, b'T' | b'F')
168 && let Some(group) = argument.group()
169 {
170 pending.push(body_stream(&group, stream.definitions.clone(), true));
171 }
172 }
173 if is_message_definition(&call.name) {
174 for argument in &call.arguments[2..] {
175 if let Some(atoms) = argument.atoms(&stream.definitions) {
176 call.message_parameters.extend(message_parameters(&atoms));
177 }
178 }
179 }
180 index.calls.insert(node.text_range().start(), call);
181 }
182 _ => {
183 if stream.active {
184 known_boundary = false;
185 }
186 }
187 }
188 }
189 }
190 index
191 }
192}
193
194fn environment_stream(node: &SyntaxNode, definitions: Arc<[u8]>, active: bool) -> Stream {
195 Stream {
196 elements: node
197 .children_with_tokens()
198 .filter(|el| !matches!(el.kind(), SyntaxKind::BEGIN | SyntaxKind::END))
199 .collect(),
200 definitions,
201 active,
202 }
203}
204
205fn toggle_may_be_an_operand(node: &SyntaxNode) -> bool {
206 if crate::semantic::define::in_reference_position(node) {
207 return true;
208 }
209 let mut previous = node.prev_sibling_or_token();
210 while let Some(element) = previous {
211 if !is_trivia(element.kind()) && element.kind() != SyntaxKind::DOC_COMMENT {
212 return element.as_node().is_some_and(|previous| {
213 previous.kind() == SyntaxKind::COMMAND
214 && read_call(previous).is_none()
215 && !command_name(previous)
216 .is_some_and(|name| expl_toggle(&format!("\\{name}")).is_some())
217 && !previous.children().any(|child| {
218 matches!(child.kind(), SyntaxKind::GROUP | SyntaxKind::OPTIONAL)
219 })
220 });
221 }
222 previous = match element {
223 SyntaxElement::Node(node) => node.prev_sibling_or_token(),
224 SyntaxElement::Token(token) => token.prev_sibling_or_token(),
225 };
226 }
227 false
228}
229
230fn is_stream_trivia(kind: SyntaxKind) -> bool {
231 is_trivia(kind)
232 || matches!(
233 kind,
234 SyntaxKind::GUARD | SyntaxKind::DOC_MARGIN | SyntaxKind::TILDE
235 )
236}
237
238fn closed_group(group: &Group) -> bool {
239 group
240 .syntax()
241 .first_child_or_token()
242 .is_some_and(|el| el.kind() == SyntaxKind::L_BRACE)
243 && group
244 .syntax()
245 .last_child_or_token()
246 .is_some_and(|el| el.kind() == SyntaxKind::R_BRACE)
247}
248
249fn body_stream(group: &Group, definitions: Arc<[u8]>, active: bool) -> Stream {
250 let mut elements: VecDeque<_> = group.syntax().children_with_tokens().collect();
251 elements.pop_front();
252 elements.pop_back();
253 Stream {
254 elements,
255 definitions,
256 active,
257 }
258}
259
260fn read_call(node: &SyntaxNode) -> Option<Call> {
264 let command = Command::cast(node.clone())?;
265 let name = command.name()?.to_string();
266 let (stem, spec) = name.rsplit_once(':')?;
267 if stem.is_empty() || stem.starts_with(':') {
268 return None;
269 }
270 let slots = expl3_slots(&name)?;
271 let mut elements: VecDeque<_> = node
272 .children_with_tokens()
273 .skip_while(|el| el.kind() != SyntaxKind::CONTROL_WORD)
274 .skip(1)
275 .filter(|el| !is_trivia(el.kind()) && el.kind() != SyntaxKind::TILDE)
276 .collect();
277 let mut arguments = Vec::new();
278 for (specifier, slot) in spec.bytes().zip(slots) {
279 let mut argument = Argument {
280 specifier,
281 elements: Vec::new(),
282 };
283 match slot {
284 Expl3Slot::Group | Expl3Slot::Branch => {
285 let el = elements.pop_front()?;
286 let group = Group::cast(el.as_node()?.clone())?;
287 if !closed_group(&group) {
288 return None;
289 }
290 argument.elements.push(el);
291 }
292 Expl3Slot::SingleToken => {
293 let el = elements.pop_front()?;
294 match &el {
295 SyntaxElement::Node(node) if node.kind() == SyntaxKind::GROUP => {
296 if !closed_group(&Group::cast(node.clone())?) {
297 return None;
298 }
299 }
300 SyntaxElement::Node(node) if node.kind() == SyntaxKind::COMMAND => {
301 if node
302 .children_with_tokens()
303 .filter(|el| !is_trivia(el.kind()))
304 .count()
305 != 1
306 {
307 return None;
308 }
309 }
310 SyntaxElement::Token(token)
311 if matches!(
312 token.kind(),
313 SyntaxKind::CONTROL_WORD | SyntaxKind::CONTROL_SYMBOL
314 ) => {}
315 SyntaxElement::Token(token)
316 if token.kind() == SyntaxKind::WORD
317 && token.text().chars().count() == 1 => {}
318 _ => return None,
319 }
320 argument.elements.push(el);
321 }
322 Expl3Slot::ParameterText => {
323 while elements.front()?.kind() != SyntaxKind::GROUP {
324 argument.elements.push(elements.pop_front()?);
325 }
326 }
327 }
328 arguments.push(argument);
329 }
330 if !elements.is_empty() {
331 return None;
332 }
333 Some(Call {
334 name,
335 arguments,
336 message_parameters: Vec::new(),
337 })
338}
339
340impl Argument {
341 fn group(&self) -> Option<Group> {
342 if self.elements.len() != 1 {
343 return None;
344 }
345 Group::cast(self.elements[0].as_node()?.clone())
346 }
347
348 pub fn name(&self) -> Option<String> {
352 match self.specifier {
353 b'N' => {
354 let mut elements = self.elements.clone();
355 if let Some(group) = self.group() {
356 elements = group
357 .syntax()
358 .children_with_tokens()
359 .filter(|el| {
360 !is_trivia(el.kind())
361 && !matches!(el.kind(), SyntaxKind::L_BRACE | SyntaxKind::R_BRACE)
362 })
363 .collect();
364 }
365 if elements.len() != 1 {
366 return None;
367 }
368 let command = Command::cast(elements[0].as_node()?.clone())?;
369 if command.syntax().children_with_tokens().count() != 1 {
370 return None;
371 }
372 Some(command.name()?.to_string())
373 }
374 b'c' => Some(self.literal()?.text),
375 _ => None,
376 }
377 }
378
379 pub fn literal(&self) -> Option<Literal> {
380 let group = self.group()?;
381 let mut text = String::new();
382 let mut range = None;
383 for el in group.syntax().children_with_tokens() {
384 if is_trivia(el.kind())
385 || matches!(el.kind(), SyntaxKind::L_BRACE | SyntaxKind::R_BRACE)
386 {
387 continue;
388 }
389 let token = el.as_token()?;
390 if token.kind() != SyntaxKind::WORD {
391 return None;
392 }
393 text.push_str(token.text());
394 range = Some(range.map_or(token.text_range(), |r: TextRange| {
395 r.cover(token.text_range())
396 }));
397 }
398 Some(Literal {
399 text,
400 range: range.unwrap_or_else(|| {
401 TextRange::empty(group.syntax().text_range().start() + TextSize::from(1))
402 }),
403 })
404 }
405
406 pub fn list(&self) -> Option<Vec<Literal>> {
407 if self.specifier != b'n' {
408 return None;
409 }
410 let atoms = self.atoms(&[])?;
411 let mut entries = Vec::new();
412 let mut text = String::new();
413 let mut range: Option<TextRange> = None;
414 for atom in atoms {
415 let AtomKind::Character(c) = atom.kind else {
416 return None;
417 };
418 if c == ',' {
419 if let Some(range) = range.take() {
420 entries.push(Literal {
421 text: std::mem::take(&mut text),
422 range,
423 });
424 }
425 } else if c.is_ascii_alphabetic() {
426 text.push(c);
427 range = Some(range.map_or(atom.range, |r| r.cover(atom.range)));
428 } else {
429 return None;
430 }
431 }
432 if let Some(range) = range {
433 entries.push(Literal { text, range });
434 }
435 Some(entries)
436 }
437
438 fn atoms(&self, definitions: &[u8]) -> Option<Vec<Atom>> {
439 let elements = if let Some(group) = self.group() {
440 body_stream(&group, Arc::from([]), true)
441 .elements
442 .into_iter()
443 .collect()
444 } else {
445 self.elements.clone()
446 };
447 let mut atoms = Vec::new();
448 for element in elements {
449 match element {
450 SyntaxElement::Token(token) => append_atoms(&token, &mut atoms),
451 SyntaxElement::Node(node) => {
452 for token in node
453 .descendants_with_tokens()
454 .filter_map(|el| el.into_token())
455 {
456 append_atoms(&token, &mut atoms);
457 }
458 }
459 }
460 }
461 for &count in definitions {
462 atoms = unescape_parameters(atoms, count)?;
463 }
464 Some(atoms)
465 }
466}
467
468pub(super) fn definition_kind(stem: &str) -> Option<bool> {
471 let (family, tail) = stem.split_once('_')?;
472 let tail = ["new", "set", "gset"]
473 .into_iter()
474 .find_map(|op| tail.strip_prefix(op))?;
475 match family {
476 "cs" if matches!(tail, "" | "_protected" | "_nopar" | "_protected_nopar") => Some(false),
477 "prg" if matches!(tail, "_conditional" | "_protected_conditional") => Some(true),
478 _ => None,
479 }
480}
481
482fn definition_body(call: &Call, definitions: &[u8]) -> Option<(Group, u8)> {
483 let (stem, spec) = call.name.rsplit_once(':')?;
484 let conditional = definition_kind(stem)?;
485 if !(if conditional {
486 matches!(spec, "Nnn" | "Npnn" | "cnn" | "cpnn")
487 } else {
488 matches!(spec, "Nn" | "Npn" | "cn" | "cpn")
489 }) {
490 return None;
491 }
492 let name = call.arguments.first()?.name()?;
493 let count = if let Some(argument) = call.arguments.iter().find(|a| a.specifier == b'p') {
494 let atoms = argument.atoms(definitions)?;
495 let mut count = 0;
496 let mut pairs = atoms.chunks_exact(2);
497 for pair in &mut pairs {
498 count += 1;
499 if count > 9
500 || pair[0].kind != AtomKind::Hash
501 || pair[1].kind != AtomKind::Character(char::from(b'0' + count))
502 {
503 return None;
504 }
505 }
506 if !pairs.remainder().is_empty() {
507 return None;
508 }
509 count
510 } else {
511 let slots = expl3_slots(&name)?;
512 if slots.len() > 9 || slots.contains(&Expl3Slot::ParameterText) {
513 return None;
514 }
515 slots.len() as u8
516 };
517 Some((call.arguments.last()?.group()?, count))
518}
519
520pub fn is_message_definition(name: &str) -> bool {
521 name.rsplit_once(':').is_some_and(|(stem, spec)| {
522 matches!(stem, "msg_new" | "msg_set" | "msg_gset") && matches!(spec, "nnn" | "nnnn")
523 })
524}
525
526pub fn is_protected_conditional(name: &str) -> bool {
527 name.rsplit_once(':').is_some_and(|(stem, spec)| {
528 matches!(
529 stem,
530 "prg_new_protected_conditional"
531 | "prg_set_protected_conditional"
532 | "prg_gset_protected_conditional"
533 ) && matches!(spec, "Nnn" | "Npnn" | "cnn" | "cpnn")
534 })
535}
536
537pub fn is_variant_generation(name: &str) -> bool {
538 matches!(
539 name,
540 "cs_generate_variant:Nn"
541 | "cs_generate_variant:cn"
542 | "prg_generate_conditional_variant:Nnn"
543 | "prg_generate_conditional_variant:cnn"
544 )
545}
546
547#[derive(Clone, Copy, PartialEq, Eq)]
548enum AtomKind {
549 Hash,
550 Character(char),
551 Other,
552 Unknown,
553}
554
555struct Atom {
556 kind: AtomKind,
557 range: TextRange,
558}
559
560fn append_atoms(token: &crate::syntax::SyntaxToken, atoms: &mut Vec<Atom>) {
561 if is_trivia(token.kind()) {
562 return;
563 }
564 if token.kind() == SyntaxKind::WORD {
565 for (offset, c) in token.text().char_indices() {
566 let start = token.text_range().start() + TextSize::from(offset as u32);
567 atoms.push(Atom {
568 kind: AtomKind::Character(c),
569 range: TextRange::at(start, TextSize::of(c)),
570 });
571 }
572 } else {
573 atoms.push(Atom {
574 kind: if token.kind() == SyntaxKind::HASH {
575 AtomKind::Hash
576 } else {
577 AtomKind::Other
578 },
579 range: token.text_range(),
580 });
581 }
582}
583
584fn unescape_parameters(atoms: Vec<Atom>, count: u8) -> Option<Vec<Atom>> {
585 let mut input = atoms.into_iter();
586 let mut output = Vec::new();
587 while let Some(atom) = input.next() {
588 if atom.kind != AtomKind::Hash {
589 output.push(atom);
590 continue;
591 }
592 let next = input.next()?;
593 let kind = match next.kind {
594 AtomKind::Hash => AtomKind::Hash,
595 AtomKind::Character(c) if c >= '1' && c <= char::from(b'0' + count) => {
596 AtomKind::Unknown
597 }
598 AtomKind::Unknown => AtomKind::Unknown,
599 _ => return None,
600 };
601 output.push(Atom {
602 kind,
603 range: atom.range.cover(next.range),
604 });
605 }
606 Some(output)
607}
608
609fn message_parameters(atoms: &[Atom]) -> Vec<(TextRange, u8)> {
610 let mut found = Vec::new();
611 let mut atoms = atoms.iter();
612 while let Some(atom) = atoms.next() {
613 if atom.kind != AtomKind::Hash {
614 continue;
615 }
616 if let Some(next) = atoms.next()
617 && let AtomKind::Character(c @ '5'..='9') = next.kind
618 {
619 found.push((atom.range.cover(next.range), c as u8 - b'0'));
620 }
621 }
622 found
623}
624
625#[cfg(test)]
626mod tests {
627 use super::*;
628 use crate::parser::parse;
629
630 fn root(body: &str) -> SyntaxNode {
631 let source = format!("\\ExplSyntaxOn\n{body}\n\\ExplSyntaxOff\n");
632 let parsed = parse(&source);
633 let root = SyntaxNode::new_root(parsed.green);
634 assert_eq!(root.text().to_string(), source);
635 root
636 }
637
638 #[test]
639 fn expl3_call_reader_preserves_expansion_letters() {
640 let root = root(r"\demo:NVncvoxef \a \b {n}{c}{v}{o}{x}{e}{f}");
641 let index = Expl3Index::build(&root);
642 let call = index
643 .calls
644 .values()
645 .find(|c| c.name == "demo:NVncvoxef")
646 .unwrap();
647 assert_eq!(
648 call.arguments
649 .iter()
650 .map(|a| a.specifier)
651 .collect::<Vec<_>>(),
652 b"NVncvoxef"
653 );
654 }
655
656 #[test]
657 fn expl3_call_reader_declines_partial_and_greedy_shapes() {
658 for source in [
659 r"\demo:Nn \a",
660 "{\\demo:Nn \\a\n\n{n}}",
661 r"\demo:w {n}",
662 r"\demo:D {n}",
663 r"\demo:Tn {a}{b}",
664 r"\demo:Nn \a {unclosed",
665 ] {
666 let root = root(source);
667 let index = Expl3Index::build(&root);
668 assert!(index.calls.is_empty(), "{source}");
669 }
670 }
671
672 #[test]
673 fn expl3_definees_and_token_lists_are_not_calls() {
674 let root = root(
675 r"\cs_new_eq:NN \msg_new:nnn \other:nnn
676 \tl_set:Nn \l_data_tl {\msg_new:nnn{a}{b}{#5}}
677 \msg_new:nnn{a}{b}{#6}",
678 );
679 let index = Expl3Index::build(&root);
680 let calls: Vec<_> = index
681 .calls
682 .values()
683 .filter(|c| c.name == "msg_new:nnn")
684 .collect();
685 assert_eq!(calls.len(), 1);
686 assert_eq!(calls[0].message_parameters[0].1, 6);
687 }
688}