Skip to main content

badness_parser/semantic/expl3/
calls.rs

1//! Conservative expl3 call reads shared by semantic lint rules and completion.
2//!
3//! The CST proves argument attachment, not execution. This index enters only
4//! recognized executable bodies, and an unresolved head stops recognition in
5//! its surrounding stream. Stored token lists and expansion wrappers stay
6//! opaque. Parameter escaping is a source-mapped view of known definition
7//! bodies; it never substitutes arguments, expands macros, or changes the tree.
8
9use std::collections::{HashMap, VecDeque};
10use std::sync::Arc;
11
12use rowan::{TextRange, TextSize};
13
14use crate::ast::{AstNode, Command, Environment, Group, command_name};
15use crate::parser::lexer::{ExplToggle, expl_toggle};
16use crate::semantic::expl3::{Expl3Slot, expl3_slots};
17use crate::syntax::{SyntaxElement, SyntaxKind, SyntaxNode, is_trivia};
18
19#[derive(Default)]
20pub struct Expl3Index {
21    calls: HashMap<TextSize, Call>,
22}
23
24pub struct Call {
25    pub name: String,
26    pub arguments: Vec<Argument>,
27    /// Actual message parameters after the enclosing definitions' hash escapes.
28    pub message_parameters: Vec<(TextRange, u8)>,
29}
30
31pub struct Argument {
32    pub specifier: u8,
33    elements: Vec<SyntaxElement>,
34}
35
36pub struct Literal {
37    pub text: String,
38    pub range: TextRange,
39}
40
41struct Stream {
42    elements: VecDeque<SyntaxElement>,
43    definitions: Arc<[u8]>,
44    active: bool,
45}
46
47impl Expl3Index {
48    /// Recognized calls in source order.
49    pub fn calls(&self) -> impl Iterator<Item = &Call> {
50        let mut calls: Vec<_> = self.calls.iter().collect();
51        calls.sort_unstable_by_key(|(offset, _)| **offset);
52        calls.into_iter().map(|(_, call)| call)
53    }
54
55    pub fn get(&self, node: &SyntaxNode) -> Option<&Call> {
56        self.calls.get(&node.text_range().start())
57    }
58
59    pub fn build(root: &SyntaxNode) -> Self {
60        let mut index = Self::default();
61        let documentation = root
62            .descendants_with_tokens()
63            .any(|el| el.kind() == SyntaxKind::DOC_MARGIN);
64        // Docstrip extracts only macrocode bodies. Documentation can itself
65        // contain live-looking expl3 examples, including syntax toggles.
66        let mut pending = if documentation {
67            root.descendants()
68                .filter_map(Environment::cast)
69                .filter(|env| {
70                    env.name()
71                        .is_some_and(|name| matches!(name.as_str(), "macrocode" | "macrocode*"))
72                })
73                .map(|env| environment_stream(env.syntax(), Arc::from([]), true))
74                .collect()
75        } else {
76            vec![Stream {
77                elements: root.children_with_tokens().collect(),
78                definitions: Arc::from([]),
79                active: false,
80            }]
81        };
82        while let Some(mut stream) = pending.pop() {
83            let mut known_boundary = true;
84            while let Some(element) = stream.elements.pop_front() {
85                let Some(node) = element.as_node() else {
86                    // A parameter or expansion token at statement position may
87                    // consume subsequent commands. Trivia cannot restore proof.
88                    if stream.active && !is_stream_trivia(element.kind()) {
89                        known_boundary = false;
90                    }
91                    continue;
92                };
93                match node.kind() {
94                    SyntaxKind::PARAGRAPH | SyntaxKind::TEXT => {
95                        for child in node
96                            .children_with_tokens()
97                            .collect::<Vec<_>>()
98                            .into_iter()
99                            .rev()
100                        {
101                            stream.elements.push_front(child);
102                        }
103                    }
104                    SyntaxKind::DOC_COMMENT => {}
105                    SyntaxKind::ENVIRONMENT => {
106                        let Some(env) = Environment::cast(node.clone()) else {
107                            continue;
108                        };
109                        let Some(name) = env.name() else { continue };
110                        if known_boundary
111                            && matches!(name.as_str(), "document" | "macrocode" | "macrocode*")
112                        {
113                            pending.push(environment_stream(
114                                node,
115                                stream.definitions.clone(),
116                                stream.active || name.starts_with("macrocode"),
117                            ));
118                        }
119                    }
120                    SyntaxKind::GROUP if known_boundary => {
121                        if let Some(group) = Group::cast(node.clone()).filter(closed_group) {
122                            pending.push(body_stream(
123                                &group,
124                                stream.definitions.clone(),
125                                stream.active,
126                            ));
127                        }
128                    }
129                    SyntaxKind::COMMAND => {
130                        let Some(name) = command_name(node) else {
131                            known_boundary &= !stream.active;
132                            continue;
133                        };
134                        if let Some(toggle) = expl_toggle(&format!("\\{name}")) {
135                            if (known_boundary || !stream.active) && !toggle_may_be_an_operand(node)
136                            {
137                                stream.active = toggle == ExplToggle::On;
138                                known_boundary = true;
139                            } else {
140                                // A referenced toggle can still have changed the
141                                // lexer mode; that does not prove execution.
142                                stream.active = true;
143                                known_boundary = false;
144                            }
145                            continue;
146                        }
147                        if !known_boundary {
148                            continue;
149                        }
150                        let Some(mut call) = read_call(node) else {
151                            known_boundary &= !stream.active;
152                            continue;
153                        };
154                        stream.active = true;
155                        if call.name.starts_with("exp_") || call.name.starts_with("use:") {
156                            // Expansion drivers can expose a head that consumes
157                            // further siblings beyond the driver's own argspec.
158                            known_boundary = false;
159                            continue;
160                        }
161                        if let Some((body, count)) = definition_body(&call, &stream.definitions) {
162                            let mut definitions = stream.definitions.to_vec();
163                            definitions.push(count);
164                            pending.push(body_stream(&body, definitions.into(), true));
165                        }
166                        for argument in &call.arguments {
167                            if matches!(argument.specifier, b'T' | b'F')
168                                && let Some(group) = argument.group()
169                            {
170                                pending.push(body_stream(&group, stream.definitions.clone(), true));
171                            }
172                        }
173                        if is_message_definition(&call.name) {
174                            for argument in &call.arguments[2..] {
175                                if let Some(atoms) = argument.atoms(&stream.definitions) {
176                                    call.message_parameters.extend(message_parameters(&atoms));
177                                }
178                            }
179                        }
180                        index.calls.insert(node.text_range().start(), call);
181                    }
182                    _ => {
183                        if stream.active {
184                            known_boundary = false;
185                        }
186                    }
187                }
188            }
189        }
190        index
191    }
192}
193
194fn environment_stream(node: &SyntaxNode, definitions: Arc<[u8]>, active: bool) -> Stream {
195    Stream {
196        elements: node
197            .children_with_tokens()
198            .filter(|el| !matches!(el.kind(), SyntaxKind::BEGIN | SyntaxKind::END))
199            .collect(),
200        definitions,
201        active,
202    }
203}
204
205fn toggle_may_be_an_operand(node: &SyntaxNode) -> bool {
206    if crate::semantic::define::in_reference_position(node) {
207        return true;
208    }
209    let mut previous = node.prev_sibling_or_token();
210    while let Some(element) = previous {
211        if !is_trivia(element.kind()) && element.kind() != SyntaxKind::DOC_COMMENT {
212            return element.as_node().is_some_and(|previous| {
213                previous.kind() == SyntaxKind::COMMAND
214                    && read_call(previous).is_none()
215                    && !command_name(previous)
216                        .is_some_and(|name| expl_toggle(&format!("\\{name}")).is_some())
217                    && !previous.children().any(|child| {
218                        matches!(child.kind(), SyntaxKind::GROUP | SyntaxKind::OPTIONAL)
219                    })
220            });
221        }
222        previous = match element {
223            SyntaxElement::Node(node) => node.prev_sibling_or_token(),
224            SyntaxElement::Token(token) => token.prev_sibling_or_token(),
225        };
226    }
227    false
228}
229
230fn is_stream_trivia(kind: SyntaxKind) -> bool {
231    is_trivia(kind)
232        || matches!(
233            kind,
234            SyntaxKind::GUARD | SyntaxKind::DOC_MARGIN | SyntaxKind::TILDE
235        )
236}
237
238fn closed_group(group: &Group) -> bool {
239    group
240        .syntax()
241        .first_child_or_token()
242        .is_some_and(|el| el.kind() == SyntaxKind::L_BRACE)
243        && group
244            .syntax()
245            .last_child_or_token()
246            .is_some_and(|el| el.kind() == SyntaxKind::R_BRACE)
247}
248
249fn body_stream(group: &Group, definitions: Arc<[u8]>, active: bool) -> Stream {
250    let mut elements: VecDeque<_> = group.syntax().children_with_tokens().collect();
251    elements.pop_front();
252    elements.pop_back();
253    Stream {
254        elements,
255        definitions,
256        active,
257    }
258}
259
260/// Read only arguments attached to this head. The parser's greedy and partial
261/// fallbacks do not prove a complete call, so sibling recovery is deliberately
262/// absent here.
263fn read_call(node: &SyntaxNode) -> Option<Call> {
264    let command = Command::cast(node.clone())?;
265    let name = command.name()?.to_string();
266    let (stem, spec) = name.rsplit_once(':')?;
267    if stem.is_empty() || stem.starts_with(':') {
268        return None;
269    }
270    let slots = expl3_slots(&name)?;
271    let mut elements: VecDeque<_> = node
272        .children_with_tokens()
273        .skip_while(|el| el.kind() != SyntaxKind::CONTROL_WORD)
274        .skip(1)
275        .filter(|el| !is_trivia(el.kind()) && el.kind() != SyntaxKind::TILDE)
276        .collect();
277    let mut arguments = Vec::new();
278    for (specifier, slot) in spec.bytes().zip(slots) {
279        let mut argument = Argument {
280            specifier,
281            elements: Vec::new(),
282        };
283        match slot {
284            Expl3Slot::Group | Expl3Slot::Branch => {
285                let el = elements.pop_front()?;
286                let group = Group::cast(el.as_node()?.clone())?;
287                if !closed_group(&group) {
288                    return None;
289                }
290                argument.elements.push(el);
291            }
292            Expl3Slot::SingleToken => {
293                let el = elements.pop_front()?;
294                match &el {
295                    SyntaxElement::Node(node) if node.kind() == SyntaxKind::GROUP => {
296                        if !closed_group(&Group::cast(node.clone())?) {
297                            return None;
298                        }
299                    }
300                    SyntaxElement::Node(node) if node.kind() == SyntaxKind::COMMAND => {
301                        if node
302                            .children_with_tokens()
303                            .filter(|el| !is_trivia(el.kind()))
304                            .count()
305                            != 1
306                        {
307                            return None;
308                        }
309                    }
310                    SyntaxElement::Token(token)
311                        if matches!(
312                            token.kind(),
313                            SyntaxKind::CONTROL_WORD | SyntaxKind::CONTROL_SYMBOL
314                        ) => {}
315                    SyntaxElement::Token(token)
316                        if token.kind() == SyntaxKind::WORD
317                            && token.text().chars().count() == 1 => {}
318                    _ => return None,
319                }
320                argument.elements.push(el);
321            }
322            Expl3Slot::ParameterText => {
323                while elements.front()?.kind() != SyntaxKind::GROUP {
324                    argument.elements.push(elements.pop_front()?);
325                }
326            }
327        }
328        arguments.push(argument);
329    }
330    if !elements.is_empty() {
331        return None;
332    }
333    Some(Call {
334        name,
335        arguments,
336        message_parameters: Vec::new(),
337    })
338}
339
340impl Argument {
341    fn group(&self) -> Option<Group> {
342        if self.elements.len() != 1 {
343            return None;
344        }
345        Group::cast(self.elements[0].as_node()?.clone())
346    }
347
348    /// Literal `N` and `c` names have the same semantic value, but require
349    /// different structural proofs. A command somewhere in a group is not
350    /// enough to establish a single control-sequence argument.
351    pub fn name(&self) -> Option<String> {
352        match self.specifier {
353            b'N' => {
354                let mut elements = self.elements.clone();
355                if let Some(group) = self.group() {
356                    elements = group
357                        .syntax()
358                        .children_with_tokens()
359                        .filter(|el| {
360                            !is_trivia(el.kind())
361                                && !matches!(el.kind(), SyntaxKind::L_BRACE | SyntaxKind::R_BRACE)
362                        })
363                        .collect();
364                }
365                if elements.len() != 1 {
366                    return None;
367                }
368                let command = Command::cast(elements[0].as_node()?.clone())?;
369                if command.syntax().children_with_tokens().count() != 1 {
370                    return None;
371                }
372                Some(command.name()?.to_string())
373            }
374            b'c' => Some(self.literal()?.text),
375            _ => None,
376        }
377    }
378
379    pub fn literal(&self) -> Option<Literal> {
380        let group = self.group()?;
381        let mut text = String::new();
382        let mut range = None;
383        for el in group.syntax().children_with_tokens() {
384            if is_trivia(el.kind())
385                || matches!(el.kind(), SyntaxKind::L_BRACE | SyntaxKind::R_BRACE)
386            {
387                continue;
388            }
389            let token = el.as_token()?;
390            if token.kind() != SyntaxKind::WORD {
391                return None;
392            }
393            text.push_str(token.text());
394            range = Some(range.map_or(token.text_range(), |r: TextRange| {
395                r.cover(token.text_range())
396            }));
397        }
398        Some(Literal {
399            text,
400            range: range.unwrap_or_else(|| {
401                TextRange::empty(group.syntax().text_range().start() + TextSize::from(1))
402            }),
403        })
404    }
405
406    pub fn list(&self) -> Option<Vec<Literal>> {
407        if self.specifier != b'n' {
408            return None;
409        }
410        let atoms = self.atoms(&[])?;
411        let mut entries = Vec::new();
412        let mut text = String::new();
413        let mut range: Option<TextRange> = None;
414        for atom in atoms {
415            let AtomKind::Character(c) = atom.kind else {
416                return None;
417            };
418            if c == ',' {
419                if let Some(range) = range.take() {
420                    entries.push(Literal {
421                        text: std::mem::take(&mut text),
422                        range,
423                    });
424                }
425            } else if c.is_ascii_alphabetic() {
426                text.push(c);
427                range = Some(range.map_or(atom.range, |r| r.cover(atom.range)));
428            } else {
429                return None;
430            }
431        }
432        if let Some(range) = range {
433            entries.push(Literal { text, range });
434        }
435        Some(entries)
436    }
437
438    fn atoms(&self, definitions: &[u8]) -> Option<Vec<Atom>> {
439        let elements = if let Some(group) = self.group() {
440            body_stream(&group, Arc::from([]), true)
441                .elements
442                .into_iter()
443                .collect()
444        } else {
445            self.elements.clone()
446        };
447        let mut atoms = Vec::new();
448        for element in elements {
449            match element {
450                SyntaxElement::Token(token) => append_atoms(&token, &mut atoms),
451                SyntaxElement::Node(node) => {
452                    for token in node
453                        .descendants_with_tokens()
454                        .filter_map(|el| el.into_token())
455                    {
456                        append_atoms(&token, &mut atoms);
457                    }
458                }
459            }
460        }
461        for &count in definitions {
462            atoms = unescape_parameters(atoms, count)?;
463        }
464        Some(atoms)
465    }
466}
467
468/// The direct creator families are a small semantic allowlist. In particular,
469/// `*_eq` and expansion variants do not establish executable replacement text.
470pub(super) fn definition_kind(stem: &str) -> Option<bool> {
471    let (family, tail) = stem.split_once('_')?;
472    let tail = ["new", "set", "gset"]
473        .into_iter()
474        .find_map(|op| tail.strip_prefix(op))?;
475    match family {
476        "cs" if matches!(tail, "" | "_protected" | "_nopar" | "_protected_nopar") => Some(false),
477        "prg" if matches!(tail, "_conditional" | "_protected_conditional") => Some(true),
478        _ => None,
479    }
480}
481
482fn definition_body(call: &Call, definitions: &[u8]) -> Option<(Group, u8)> {
483    let (stem, spec) = call.name.rsplit_once(':')?;
484    let conditional = definition_kind(stem)?;
485    if !(if conditional {
486        matches!(spec, "Nnn" | "Npnn" | "cnn" | "cpnn")
487    } else {
488        matches!(spec, "Nn" | "Npn" | "cn" | "cpn")
489    }) {
490        return None;
491    }
492    let name = call.arguments.first()?.name()?;
493    let count = if let Some(argument) = call.arguments.iter().find(|a| a.specifier == b'p') {
494        let atoms = argument.atoms(definitions)?;
495        let mut count = 0;
496        let mut pairs = atoms.chunks_exact(2);
497        for pair in &mut pairs {
498            count += 1;
499            if count > 9
500                || pair[0].kind != AtomKind::Hash
501                || pair[1].kind != AtomKind::Character(char::from(b'0' + count))
502            {
503                return None;
504            }
505        }
506        if !pairs.remainder().is_empty() {
507            return None;
508        }
509        count
510    } else {
511        let slots = expl3_slots(&name)?;
512        if slots.len() > 9 || slots.contains(&Expl3Slot::ParameterText) {
513            return None;
514        }
515        slots.len() as u8
516    };
517    Some((call.arguments.last()?.group()?, count))
518}
519
520pub fn is_message_definition(name: &str) -> bool {
521    name.rsplit_once(':').is_some_and(|(stem, spec)| {
522        matches!(stem, "msg_new" | "msg_set" | "msg_gset") && matches!(spec, "nnn" | "nnnn")
523    })
524}
525
526pub fn is_protected_conditional(name: &str) -> bool {
527    name.rsplit_once(':').is_some_and(|(stem, spec)| {
528        matches!(
529            stem,
530            "prg_new_protected_conditional"
531                | "prg_set_protected_conditional"
532                | "prg_gset_protected_conditional"
533        ) && matches!(spec, "Nnn" | "Npnn" | "cnn" | "cpnn")
534    })
535}
536
537pub fn is_variant_generation(name: &str) -> bool {
538    matches!(
539        name,
540        "cs_generate_variant:Nn"
541            | "cs_generate_variant:cn"
542            | "prg_generate_conditional_variant:Nnn"
543            | "prg_generate_conditional_variant:cnn"
544    )
545}
546
547#[derive(Clone, Copy, PartialEq, Eq)]
548enum AtomKind {
549    Hash,
550    Character(char),
551    Other,
552    Unknown,
553}
554
555struct Atom {
556    kind: AtomKind,
557    range: TextRange,
558}
559
560fn append_atoms(token: &crate::syntax::SyntaxToken, atoms: &mut Vec<Atom>) {
561    if is_trivia(token.kind()) {
562        return;
563    }
564    if token.kind() == SyntaxKind::WORD {
565        for (offset, c) in token.text().char_indices() {
566            let start = token.text_range().start() + TextSize::from(offset as u32);
567            atoms.push(Atom {
568                kind: AtomKind::Character(c),
569                range: TextRange::at(start, TextSize::of(c)),
570            });
571        }
572    } else {
573        atoms.push(Atom {
574            kind: if token.kind() == SyntaxKind::HASH {
575                AtomKind::Hash
576            } else {
577                AtomKind::Other
578            },
579            range: token.text_range(),
580        });
581    }
582}
583
584fn unescape_parameters(atoms: Vec<Atom>, count: u8) -> Option<Vec<Atom>> {
585    let mut input = atoms.into_iter();
586    let mut output = Vec::new();
587    while let Some(atom) = input.next() {
588        if atom.kind != AtomKind::Hash {
589            output.push(atom);
590            continue;
591        }
592        let next = input.next()?;
593        let kind = match next.kind {
594            AtomKind::Hash => AtomKind::Hash,
595            AtomKind::Character(c) if c >= '1' && c <= char::from(b'0' + count) => {
596                AtomKind::Unknown
597            }
598            AtomKind::Unknown => AtomKind::Unknown,
599            _ => return None,
600        };
601        output.push(Atom {
602            kind,
603            range: atom.range.cover(next.range),
604        });
605    }
606    Some(output)
607}
608
609fn message_parameters(atoms: &[Atom]) -> Vec<(TextRange, u8)> {
610    let mut found = Vec::new();
611    let mut atoms = atoms.iter();
612    while let Some(atom) = atoms.next() {
613        if atom.kind != AtomKind::Hash {
614            continue;
615        }
616        if let Some(next) = atoms.next()
617            && let AtomKind::Character(c @ '5'..='9') = next.kind
618        {
619            found.push((atom.range.cover(next.range), c as u8 - b'0'));
620        }
621    }
622    found
623}
624
625#[cfg(test)]
626mod tests {
627    use super::*;
628    use crate::parser::parse;
629
630    fn root(body: &str) -> SyntaxNode {
631        let source = format!("\\ExplSyntaxOn\n{body}\n\\ExplSyntaxOff\n");
632        let parsed = parse(&source);
633        let root = SyntaxNode::new_root(parsed.green);
634        assert_eq!(root.text().to_string(), source);
635        root
636    }
637
638    #[test]
639    fn expl3_call_reader_preserves_expansion_letters() {
640        let root = root(r"\demo:NVncvoxef \a \b {n}{c}{v}{o}{x}{e}{f}");
641        let index = Expl3Index::build(&root);
642        let call = index
643            .calls
644            .values()
645            .find(|c| c.name == "demo:NVncvoxef")
646            .unwrap();
647        assert_eq!(
648            call.arguments
649                .iter()
650                .map(|a| a.specifier)
651                .collect::<Vec<_>>(),
652            b"NVncvoxef"
653        );
654    }
655
656    #[test]
657    fn expl3_call_reader_declines_partial_and_greedy_shapes() {
658        for source in [
659            r"\demo:Nn \a",
660            "{\\demo:Nn \\a\n\n{n}}",
661            r"\demo:w {n}",
662            r"\demo:D {n}",
663            r"\demo:Tn {a}{b}",
664            r"\demo:Nn \a {unclosed",
665        ] {
666            let root = root(source);
667            let index = Expl3Index::build(&root);
668            assert!(index.calls.is_empty(), "{source}");
669        }
670    }
671
672    #[test]
673    fn expl3_definees_and_token_lists_are_not_calls() {
674        let root = root(
675            r"\cs_new_eq:NN \msg_new:nnn \other:nnn
676            \tl_set:Nn \l_data_tl {\msg_new:nnn{a}{b}{#5}}
677            \msg_new:nnn{a}{b}{#6}",
678        );
679        let index = Expl3Index::build(&root);
680        let calls: Vec<_> = index
681            .calls
682            .values()
683            .filter(|c| c.name == "msg_new:nnn")
684            .collect();
685        assert_eq!(calls.len(), 1);
686        assert_eq!(calls[0].message_parameters[0].1, 6);
687    }
688}