Skip to main content

badness_parser/semantic/
expl3.rs

1//! The expl3 call-site model: **argspec arity** for expl3 function names, and
2//! the **statement segmentation** built on it.
3//!
4//! Two halves, both semantics layered on the syntax tree (like
5//! [`define`](super::define)'s definition scan): [`expl3_slots`] derives
6//! per-slot arity from the letters after the final `:` in `\cs_new:Npn`,
7//! `\tl_if_empty:nTF`, …, and [`segment_expl_statements`] applies it to an
8//! in-region element stream to produce the statement model the formatter's
9//! expl3 layout consumes. Neither builds `Ir` or touches layout policy — a
10//! wrong answer here can only produce ugly formatting downstream, never a
11//! wrong tree or a lost byte.
12//!
13//! Like [`xparse`](super::xparse), the argspec is parsed rather than executed:
14//! each letter describes an argument's call-site shape. No
15//! signature database is involved: the name string alone carries the spec, so
16//! there is nothing to curate and nothing to drift. Only meaningful inside an
17//! expl3 region, where `:`/`_` are catcode-11 and the whole name lexes as one
18//! `CONTROL_WORD` — callers of the segmentation guarantee the stream is
19//! in-region (out-of-region, colon names lex split and everything degrades to
20//! the fallback).
21//!
22//! The letter-by-letter model (interface3's argument specifiers):
23//!
24//! - `N`, `V` → [`Expl3Slot::SingleToken`]: one token, typically a control
25//!   sequence (`V` differs from `N` only in *expansion*, not call-site shape).
26//! - `n`, `c`, `v`, `o`, `x`, `e`, `f` → [`Expl3Slot::Group`]: one braced
27//!   `{…}` group (again, the letters differ only in how the material is
28//!   processed, which we never model).
29//! - `T`, `F` → [`Expl3Slot::Branch`]: a braced conditional branch. Sanctioned
30//!   only as a *trailing* run — in a standard argspec `T`/`F` are always last,
31//!   so a mid-spec `T`/`F` is treated as unknown.
32//! - `p` → [`Expl3Slot::ParameterText`]: TeX parameter text (`#1#2…`), which
33//!   has no fixed token count but a static *end*: TeX's own rule that the
34//!   parameter text runs to the first explicit `{`. The consumer scans by that
35//!   shape.
36//! - `w` (arbitrary delimiters) and `D` (kernel primitive) have no lexically
37//!   derivable call-site shape → the whole name is unrecognized (`None`), as is
38//!   any unknown letter (including one added to expl3 after this list was
39//!   written — new letters degrade to unrecognized, never to a wrong arity).
40
41pub mod calls;
42pub mod mode;
43pub mod symbols;
44pub mod variants;
45
46use std::collections::VecDeque;
47
48use rowan::TextRange;
49
50use crate::ast::command_name;
51use crate::parser::lexer::expl_toggle;
52use crate::syntax::{SyntaxElement, SyntaxKind, SyntaxNode, is_collapsible_trivia, is_param_digit};
53
54/// The call-site shape of one expl3 argument slot, derived from an argspec letter.
55#[derive(Debug, Clone, Copy, PartialEq, Eq)]
56pub enum Expl3Slot {
57    /// `N`, `V`: exactly one token, typically a control sequence.
58    SingleToken,
59    /// `n`, `c`, `v`, `o`, `x`, `e`, `f`: one braced `{…}` group.
60    Group,
61    /// `T`, `F`: a braced conditional branch (a [`Group`](Expl3Slot::Group) a
62    /// consumer may lay out specially).
63    Branch,
64    /// `p`: TeX parameter text — the tokens up to (not including) the next
65    /// explicit `{`.
66    ParameterText,
67}
68
69/// The argument slots of an expl3 function name, read from its argspec suffix
70/// (the substring after the *final* `:`), or `None` when the name has no
71/// derivable call-site arity.
72///
73/// `Some` iff the name contains a `:` and every suffix letter is a fixed-shape
74/// letter per the module docs; an empty suffix (`\scan_stop:`, `\group_end:`)
75/// is `Some(vec![])` — a recognized zero-argument call. `None` for a colonless
76/// name (`\def`, `\@ifpackageloaded`), or a spec containing `w`, `D`, a
77/// mid-spec `T`/`F`, or any unknown letter.
78pub fn expl3_slots(name: &str) -> Option<Vec<Expl3Slot>> {
79    let argspec = name.rsplit_once(':')?.1;
80    let chars: Vec<char> = argspec.chars().collect();
81    let branches = chars
82        .iter()
83        .rev()
84        .take_while(|c| matches!(c, 'T' | 'F'))
85        .count();
86    let mut slots = Vec::with_capacity(chars.len());
87    for c in &chars[..chars.len() - branches] {
88        // `T`/`F` never match here, so a *mid*-spec `T`/`F` (nonstandard) falls
89        // through to unknown.
90        slots.push(match c {
91            'N' | 'V' => Expl3Slot::SingleToken,
92            'n' | 'c' | 'v' | 'o' | 'x' | 'e' | 'f' => Expl3Slot::Group,
93            'p' => Expl3Slot::ParameterText,
94            _ => return None,
95        });
96    }
97    slots.extend(std::iter::repeat_n(Expl3Slot::Branch, branches));
98    Some(slots)
99}
100
101/// The number of trailing `T`/`F` branch arguments of an expl3 conditional, read
102/// from the command *name*'s argspec (the substring after the final `:`).
103/// `\tl_if_empty:nTF` → `Some(2)`, `\bool_if:nT`/`:nF` → `Some(1)`; `None` for any
104/// name without a `:`-argspec ending in `T`/`F` — a non-conditional expl3 function
105/// (`\seq_new:N`), or a LaTeX2e command with no colon (`\@ifpackageloaded`). In an
106/// expl3 argspec `T`/`F` denote *only* the true/false branch slots, so a trailing
107/// `T`/`F` run is exactly the branch count.
108///
109/// Deliberately **not** derived from [`expl3_slots`]: this counts the raw
110/// trailing run, so a name whose *earlier* letters make the arity unrecognized
111/// (a hypothetical `:wTF` shape) still reports its branches — the conditional
112/// layout keys on the branches alone and must not regress when the full arity
113/// model bows out.
114pub fn conditional_branches(name: &str) -> Option<usize> {
115    let argspec = name.rsplit_once(':')?.1;
116    let n = argspec
117        .chars()
118        .rev()
119        .take_while(|c| *c == 'T' || *c == 'F')
120        .count();
121    (n > 0).then_some(n)
122}
123
124#[cfg(test)]
125mod tests {
126    use super::*;
127    use Expl3Slot::*;
128
129    #[test]
130    fn slots_read_from_name_suffix() {
131        assert_eq!(
132            expl3_slots("cs_new:Npn"),
133            Some(vec![SingleToken, ParameterText, Group])
134        );
135        assert_eq!(
136            expl3_slots("str_if_eq:nnTF"),
137            Some(vec![Group, Group, Branch, Branch])
138        );
139        assert_eq!(
140            expl3_slots("prop_get:NnNTF"),
141            Some(vec![SingleToken, Group, SingleToken, Branch, Branch])
142        );
143        assert_eq!(expl3_slots("tl_set:Nn"), Some(vec![SingleToken, Group]));
144        assert_eq!(
145            expl3_slots("exp_args:NNo"),
146            Some(vec![SingleToken, SingleToken, Group])
147        );
148        assert_eq!(expl3_slots("tl_set:Nv"), Some(vec![SingleToken, Group]));
149        assert_eq!(expl3_slots("use:c"), Some(vec![Group]));
150        assert_eq!(expl3_slots("tl_set:Nx"), Some(vec![SingleToken, Group]));
151    }
152
153    #[test]
154    fn zero_argument_names_are_recognized() {
155        assert_eq!(expl3_slots("scan_stop:"), Some(vec![]));
156        assert_eq!(expl3_slots("group_begin:"), Some(vec![]));
157        assert_eq!(expl3_slots("prg_return_true:"), Some(vec![]));
158    }
159
160    #[test]
161    fn underivable_specs_are_unrecognized() {
162        assert_eq!(expl3_slots("use_none_delimit_by_q_stop:w"), None);
163        assert_eq!(expl3_slots("exp_after:wN"), None);
164        assert_eq!(expl3_slots("tex_relax:D"), None);
165        assert_eq!(expl3_slots("odd:TnF"), None);
166        assert_eq!(expl3_slots("odd:nZn"), None);
167    }
168
169    #[test]
170    fn colonless_names_are_unrecognized() {
171        assert_eq!(expl3_slots("def"), None);
172        assert_eq!(expl3_slots("@ifpackageloaded"), None);
173        assert_eq!(expl3_slots("IfBooleanTF"), None);
174        assert_eq!(expl3_slots("l_tmpa_tl"), None);
175    }
176
177    #[test]
178    fn exp_internal_drivers() {
179        assert_eq!(expl3_slots("::n"), Some(vec![Group]));
180        assert_eq!(expl3_slots(":::"), Some(vec![]));
181    }
182
183    #[test]
184    fn conditional_branches_read_from_name_suffix() {
185        assert_eq!(conditional_branches("tl_if_empty:nTF"), Some(2));
186        assert_eq!(conditional_branches("bool_if:nT"), Some(1));
187        assert_eq!(conditional_branches("bool_if:nF"), Some(1));
188        assert_eq!(conditional_branches("str_if_eq:nnTF"), Some(2));
189        assert_eq!(conditional_branches("int_compare:nNnTF"), Some(2));
190        assert_eq!(conditional_branches("seq_map_inline:Nn"), None);
191        assert_eq!(conditional_branches("prg_return_true:"), None);
192        assert_eq!(conditional_branches("tl_new:N"), None);
193        assert_eq!(conditional_branches("@ifpackageloaded"), None);
194        assert_eq!(conditional_branches("IfBooleanTF"), None);
195    }
196
197    #[test]
198    fn branches_survive_underivable_arity() {
199        assert_eq!(expl3_slots("odd_if:wTF"), None);
200        assert_eq!(conditional_branches("odd_if:wTF"), Some(2));
201    }
202}
203
204/// Statement boundaries and attachment flags for an expl3 element stream.
205pub struct StatementMap {
206    flags: Vec<ElementFlags>,
207}
208
209#[derive(Clone, Copy, Default)]
210struct ElementFlags(u8);
211
212impl ElementFlags {
213    const BOUNDARY_AFTER: u8 = 1 << 0;
214    const GLUE_BEFORE: u8 = 1 << 1;
215    const GLUED: u8 = 1 << 2;
216    const FALLBACK: u8 = 1 << 3;
217
218    fn contains(self, flag: u8) -> bool {
219        self.0 & flag != 0
220    }
221
222    fn insert(&mut self, flag: u8) {
223        self.0 |= flag;
224    }
225}
226
227impl StatementMap {
228    pub fn boundary_after(&self, idx: usize) -> bool {
229        self.flags
230            .get(idx)
231            .is_some_and(|flags| flags.contains(ElementFlags::BOUNDARY_AFTER))
232    }
233
234    pub fn glue_before(&self, idx: usize) -> bool {
235        self.flags
236            .get(idx)
237            .is_some_and(|flags| flags.contains(ElementFlags::GLUE_BEFORE))
238    }
239
240    pub fn is_glued(&self, idx: usize) -> bool {
241        self.flags
242            .get(idx)
243            .is_some_and(|flags| flags.contains(ElementFlags::GLUED))
244    }
245
246    pub fn is_fallback(&self, idx: usize) -> bool {
247        self.flags
248            .get(idx)
249            .is_some_and(|flags| flags.contains(ElementFlags::FALLBACK))
250    }
251}
252
253/// Segments an expl3 element stream into statements.
254pub fn segment_expl_statements(elements: &[SyntaxElement]) -> StatementMap {
255    let mut flags = vec![ElementFlags::default(); elements.len()];
256    let mut i = 0;
257    while i < elements.len() {
258        match &elements[i] {
259            SyntaxElement::Token(t) if is_collapsible_trivia(t.kind()) => i += 1,
260            SyntaxElement::Token(t)
261                if matches!(
262                    t.kind(),
263                    SyntaxKind::COMMENT | SyntaxKind::GUARD | SyntaxKind::DOC_MARGIN
264                ) =>
265            {
266                if followed_by_newline(elements, i) {
267                    flags[i].insert(ElementFlags::BOUNDARY_AFTER);
268                }
269                i += 1;
270            }
271            SyntaxElement::Node(n) if n.kind() == SyntaxKind::COMMAND => {
272                match expl3_unit(elements, i) {
273                    Some(unit) => {
274                        let end = unit.last;
275                        let full = absorb_trailing_junk(elements, end);
276                        if full > end {
277                            for flags in &mut flags[i..=full] {
278                                flags.insert(ElementFlags::GLUED);
279                            }
280                        }
281                        flags[full].insert(ElementFlags::BOUNDARY_AFTER);
282                        i = full + 1;
283                    }
284                    None => i = fallback_line(elements, i, &mut flags),
285                }
286            }
287            _ => i = fallback_line(elements, i, &mut flags),
288        }
289    }
290    StatementMap { flags }
291}
292
293fn node_is_expl_toggle(node: &SyntaxNode) -> bool {
294    node.children_with_tokens()
295        .filter_map(|el| el.into_token())
296        .find(|t| t.kind() == SyntaxKind::CONTROL_WORD)
297        .is_some_and(|t| expl_toggle(t.text()).is_some())
298}
299
300fn is_recognized_head(node: &SyntaxNode) -> bool {
301    node.kind() == SyntaxKind::COMMAND
302        && (node_is_expl_toggle(node)
303            || command_name(node).is_some_and(|name| expl3_slots(&name).is_some()))
304}
305
306fn followed_by_newline(elements: &[SyntaxElement], idx: usize) -> bool {
307    for element in &elements[idx + 1..] {
308        match element {
309            SyntaxElement::Token(t) if t.kind() == SyntaxKind::WHITESPACE => {}
310            SyntaxElement::Token(t) if t.kind() == SyntaxKind::NEWLINE => return true,
311            _ => return false,
312        }
313    }
314    true
315}
316
317fn fallback_line(elements: &[SyntaxElement], start: usize, flags: &mut [ElementFlags]) -> usize {
318    let mut last = start;
319    let mut j = start;
320    while j < elements.len() {
321        match &elements[j] {
322            SyntaxElement::Token(t) if is_collapsible_trivia(t.kind()) => {
323                if t.kind() == SyntaxKind::NEWLINE {
324                    flags[last].insert(ElementFlags::BOUNDARY_AFTER);
325                    for flags in &mut flags[start..=last] {
326                        flags.insert(ElementFlags::FALLBACK);
327                    }
328                    return j;
329                }
330                j += 1;
331            }
332            element => {
333                if j > start
334                    && let SyntaxElement::Node(n) = element
335                    && is_recognized_head(n)
336                {
337                    flags[j].insert(ElementFlags::GLUE_BEFORE);
338                }
339                last = j;
340                j += 1;
341                if let SyntaxElement::Node(n) = element
342                    && n.kind() == SyntaxKind::COMMAND
343                    && node_carries_bare_line_break(n)
344                {
345                    flags[last].insert(ElementFlags::BOUNDARY_AFTER);
346                    for flags in &mut flags[start..=last] {
347                        flags.insert(ElementFlags::FALLBACK);
348                    }
349                    return j;
350                }
351            }
352        }
353    }
354    flags[last].insert(ElementFlags::BOUNDARY_AFTER);
355    for flags in &mut flags[start..=last] {
356        flags.insert(ElementFlags::FALLBACK);
357    }
358    elements.len()
359}
360
361fn node_carries_bare_line_break(node: &SyntaxNode) -> bool {
362    let mut after_newline = false;
363    for child in node.children_with_tokens() {
364        match &child {
365            SyntaxElement::Token(t) if t.kind() == SyntaxKind::NEWLINE => after_newline = true,
366            SyntaxElement::Token(t) if is_collapsible_trivia(t.kind()) => {}
367            SyntaxElement::Node(n)
368                if matches!(n.kind(), SyntaxKind::GROUP | SyntaxKind::OPTIONAL) =>
369            {
370                after_newline = false;
371            }
372            _ => {
373                if after_newline {
374                    return true;
375                }
376            }
377        }
378    }
379    false
380}
381
382fn absorb_trailing_junk(elements: &[SyntaxElement], end: usize) -> usize {
383    let mut end = end;
384    let mut j = end + 1;
385    while j < elements.len() {
386        match &elements[j] {
387            SyntaxElement::Token(t) if is_collapsible_trivia(t.kind()) => {
388                if t.kind() == SyntaxKind::NEWLINE {
389                    break;
390                }
391                j += 1;
392            }
393            SyntaxElement::Token(t) if t.kind() == SyntaxKind::COMMENT => {
394                end = j;
395                break;
396            }
397            SyntaxElement::Token(t)
398                if matches!(t.kind(), SyntaxKind::GUARD | SyntaxKind::DOC_MARGIN) =>
399            {
400                break;
401            }
402            SyntaxElement::Node(n) if n.kind() == SyntaxKind::GROUP => break,
403            SyntaxElement::Node(n) if is_recognized_head(n) => {
404                break;
405            }
406            _ => {
407                end = j;
408                j += 1;
409            }
410        }
411    }
412    end
413}
414
415enum Stop {
416    End,
417    Abort,
418}
419
420fn consume_unit(
421    elements: &[SyntaxElement],
422    head_idx: usize,
423    slots: &[Expl3Slot],
424) -> Option<Expl3Unit> {
425    let head = elements[head_idx].as_node()?;
426    let mut cur = UnitCursor::new(elements, head_idx, head);
427    let mut branches = Vec::new();
428    let mut complete = true;
429    for slot in slots {
430        let took = match slot {
431            Expl3Slot::SingleToken => cur.take_single_token(),
432            Expl3Slot::Group => cur.take_group().map(|_| ()),
433            Expl3Slot::Branch => cur.take_group().map(|el| branches.push(el.text_range())),
434            Expl3Slot::ParameterText => cur.take_parameter_text(),
435        };
436        match took {
437            Ok(()) => {}
438            Err(Stop::End) => {
439                complete = false;
440                break;
441            }
442            Err(Stop::Abort) => return None,
443        }
444    }
445    cur.extend_over_attachable_tail();
446    Some(Expl3Unit {
447        last: cur.last_sib,
448        branches: if complete { branches } else { Vec::new() },
449    })
450}
451
452#[derive(Debug, Clone, PartialEq, Eq)]
453/// The span and conditional branches of one parsed expl3 unit.
454pub struct Expl3Unit {
455    pub last: usize,
456    pub branches: Vec<TextRange>,
457}
458
459/// Resolves the expl3 unit beginning at `head_idx`.
460pub fn expl3_unit(elements: &[SyntaxElement], head_idx: usize) -> Option<Expl3Unit> {
461    let node = elements.get(head_idx)?.as_node()?;
462    if !is_recognized_head(node) {
463        return None;
464    }
465    let slots = if node_is_expl_toggle(node) {
466        Vec::new()
467    } else {
468        expl3_slots(&command_name(node)?)?
469    };
470    consume_unit(elements, head_idx, &slots)
471}
472
473struct UnitCursor<'a> {
474    elements: &'a [SyntaxElement],
475    queue: VecDeque<SyntaxElement>,
476    sib: usize,
477    last_sib: usize,
478    peeked: Option<(SyntaxElement, Option<usize>)>,
479    chain: bool,
480}
481
482impl<'a> UnitCursor<'a> {
483    fn new(elements: &'a [SyntaxElement], head_idx: usize, head: &SyntaxNode) -> Self {
484        let mut cur = UnitCursor {
485            elements,
486            queue: VecDeque::new(),
487            sib: head_idx + 1,
488            last_sib: head_idx,
489            peeked: None,
490            chain: true,
491        };
492        cur.queue_children_after_name(head, false);
493        cur
494    }
495
496    fn queue_children_after_name(&mut self, node: &SyntaxNode, front: bool) {
497        let mut seen_name = false;
498        let mut after: Vec<SyntaxElement> = Vec::new();
499        for child in node.children_with_tokens() {
500            if seen_name {
501                after.push(child);
502            } else if matches!(
503                child.kind(),
504                SyntaxKind::CONTROL_WORD | SyntaxKind::CONTROL_SYMBOL
505            ) {
506                seen_name = true;
507            }
508        }
509        if front {
510            for el in after.into_iter().rev() {
511                self.queue.push_front(el);
512            }
513        } else {
514            self.queue.extend(after);
515        }
516    }
517
518    fn peek(&mut self) -> Result<&SyntaxElement, Stop> {
519        if self.peeked.is_none() {
520            self.peeked = Some(self.advance()?);
521        }
522        Ok(&self.peeked.as_ref().expect("just filled").0)
523    }
524
525    fn bump(&mut self) -> Result<SyntaxElement, Stop> {
526        let (el, sib_idx) = match self.peeked.take() {
527            Some(peeked) => peeked,
528            None => self.advance()?,
529        };
530        if let Some(idx) = sib_idx {
531            self.last_sib = idx;
532        }
533        Ok(el)
534    }
535
536    fn advance(&mut self) -> Result<(SyntaxElement, Option<usize>), Stop> {
537        let mut gap_newlines = 0usize;
538        loop {
539            let (el, sib_idx) = if let Some(el) = self.queue.pop_front() {
540                (el, None)
541            } else {
542                let Some(el) = self.elements.get(self.sib) else {
543                    return Err(Stop::Abort);
544                };
545                if let SyntaxElement::Token(t) = el
546                    && t.kind() == SyntaxKind::NEWLINE
547                    && gap_newlines >= 1
548                {
549                    return Err(Stop::End);
550                }
551                let idx = self.sib;
552                self.sib += 1;
553                (el.clone(), Some(idx))
554            };
555            match &el {
556                SyntaxElement::Token(t) if is_collapsible_trivia(t.kind()) => {
557                    if t.kind() == SyntaxKind::NEWLINE {
558                        gap_newlines += 1;
559                        if gap_newlines >= 2 {
560                            return Err(Stop::End);
561                        }
562                    }
563                }
564                SyntaxElement::Token(t) if t.kind() == SyntaxKind::COMMENT => {}
565                SyntaxElement::Token(t) if t.kind() == SyntaxKind::TILDE => {}
566                SyntaxElement::Token(t)
567                    if matches!(t.kind(), SyntaxKind::GUARD | SyntaxKind::DOC_MARGIN) =>
568                {
569                    return Err(Stop::Abort);
570                }
571                _ => return Ok((el, sib_idx)),
572            }
573        }
574    }
575
576    fn take_single_token(&mut self) -> Result<(), Stop> {
577        let el = self.bump()?;
578        match &el {
579            SyntaxElement::Token(t)
580                if matches!(
581                    t.kind(),
582                    SyntaxKind::CONTROL_WORD | SyntaxKind::CONTROL_SYMBOL
583                ) =>
584            {
585                self.chain = false;
586                Ok(())
587            }
588            SyntaxElement::Token(t)
589                if t.kind() == SyntaxKind::WORD && t.text().chars().count() == 1 =>
590            {
591                self.chain = false;
592                Ok(())
593            }
594            SyntaxElement::Token(t) if t.kind() == SyntaxKind::HASH => {
595                self.chain = false;
596                loop {
597                    let next = self.bump()?;
598                    match &next {
599                        SyntaxElement::Token(t) if t.kind() == SyntaxKind::HASH => {}
600                        SyntaxElement::Token(t)
601                            if t.kind() == SyntaxKind::WORD && is_param_digit(t) =>
602                        {
603                            return Ok(());
604                        }
605                        _ => return Err(Stop::Abort),
606                    }
607                }
608            }
609            SyntaxElement::Node(n) if n.kind() == SyntaxKind::COMMAND => {
610                self.queue_children_after_name(n, true);
611                self.chain = true;
612                Ok(())
613            }
614            SyntaxElement::Node(n) if n.kind() == SyntaxKind::GROUP => Ok(()),
615            _ => Err(Stop::Abort),
616        }
617    }
618
619    fn take_group(&mut self) -> Result<SyntaxElement, Stop> {
620        let el = self.bump()?;
621        match &el {
622            SyntaxElement::Node(n) if n.kind() == SyntaxKind::GROUP => Ok(el),
623            _ => Err(Stop::Abort),
624        }
625    }
626
627    fn take_parameter_text(&mut self) -> Result<(), Stop> {
628        loop {
629            if let SyntaxElement::Node(n) = self.peek()?
630                && n.kind() == SyntaxKind::GROUP
631            {
632                return Ok(());
633            }
634            let el = self.bump()?;
635            match &el {
636                SyntaxElement::Token(_) => self.chain = false,
637                SyntaxElement::Node(n) if n.kind() == SyntaxKind::COMMAND => {
638                    self.queue_children_after_name(n, true);
639                    self.chain = true;
640                }
641                SyntaxElement::Node(n) if n.kind() == SyntaxKind::OPTIONAL => {}
642                _ => return Err(Stop::Abort),
643            }
644        }
645    }
646
647    fn extend_over_attachable_tail(&mut self) {
648        if let Some((el, sib_idx)) = self.peeked.take()
649            && sib_idx.is_none()
650        {
651            self.update_chain(&el);
652        }
653        while let Some(el) = self.queue.pop_front() {
654            self.update_chain(&el);
655        }
656        if !self.chain {
657            return;
658        }
659        let mut newlines = 0usize;
660        let mut i = self.last_sib + 1;
661        while let Some(el) = self.elements.get(i) {
662            match el {
663                SyntaxElement::Token(t) => match t.kind() {
664                    SyntaxKind::NEWLINE => {
665                        newlines += 1;
666                        if newlines >= 2 {
667                            return;
668                        }
669                    }
670                    SyntaxKind::COMMENT => newlines = 0,
671                    SyntaxKind::WHITESPACE | SyntaxKind::GUARD | SyntaxKind::DOC_MARGIN => {}
672                    _ => return,
673                },
674                SyntaxElement::Node(n)
675                    if matches!(n.kind(), SyntaxKind::GROUP | SyntaxKind::OPTIONAL) =>
676                {
677                    self.last_sib = i;
678                    newlines = 0;
679                }
680                SyntaxElement::Node(_) => return,
681            }
682            i += 1;
683        }
684    }
685
686    fn update_chain(&mut self, el: &SyntaxElement) {
687        match el {
688            SyntaxElement::Node(n) if n.kind() == SyntaxKind::COMMAND => self.chain = true,
689            SyntaxElement::Node(n)
690                if matches!(n.kind(), SyntaxKind::GROUP | SyntaxKind::OPTIONAL) => {}
691            SyntaxElement::Token(t)
692                if is_collapsible_trivia(t.kind())
693                    || matches!(
694                        t.kind(),
695                        SyntaxKind::COMMENT
696                            | SyntaxKind::TILDE
697                            | SyntaxKind::GUARD
698                            | SyntaxKind::DOC_MARGIN
699                    ) => {}
700            _ => self.chain = false,
701        }
702    }
703}
704
705#[cfg(test)]
706mod segmentation_tests {
707    use super::*;
708    use crate::parser::parse;
709    use crate::syntax::SyntaxNode;
710
711    fn statements(src: &str) -> Vec<String> {
712        let parsed = parse(src);
713        assert!(parsed.errors.is_empty(), "test source should parse cleanly");
714        let root = SyntaxNode::new_root(parsed.green);
715        let para = root
716            .children()
717            .find(|n| n.kind() == SyntaxKind::PARAGRAPH)
718            .expect("a paragraph");
719        let elements: Vec<SyntaxElement> = para.children_with_tokens().collect();
720        statement_texts(&elements)
721    }
722
723    fn statement_texts(elements: &[SyntaxElement]) -> Vec<String> {
724        let map = segment_expl_statements(elements);
725        let mut out = Vec::new();
726        let mut cur = String::new();
727        for (i, el) in elements.iter().enumerate() {
728            cur.push_str(&el.to_string());
729            if map.boundary_after(i) {
730                let text = normalize(&cur);
731                if !text.is_empty() {
732                    out.push(text);
733                }
734                cur.clear();
735            }
736        }
737        let tail = normalize(&cur);
738        if !tail.is_empty() {
739            out.push(tail);
740        }
741        out
742    }
743
744    fn normalize(s: &str) -> String {
745        s.split_whitespace().collect::<Vec<_>>().join(" ")
746    }
747
748    #[test]
749    fn statements_are_structural_units() {
750        let got = statements(
751            "\\ExplSyntaxOn\n\\tl_set:Nn \\l_a\n  { x }\n\\group_begin:\n\\ExplSyntaxOff\n",
752        );
753        assert_eq!(
754            got,
755            vec![
756                "\\ExplSyntaxOn",
757                "\\tl_set:Nn \\l_a { x }",
758                "\\group_begin:",
759                "\\ExplSyntaxOff",
760            ]
761        );
762    }
763
764    #[test]
765    fn same_line_calls_split() {
766        let got =
767            statements("\\ExplSyntaxOn\n\\group_begin: \\int_zero:N \\l_a\n\\ExplSyntaxOff\n");
768        assert_eq!(
769            got,
770            vec![
771                "\\ExplSyntaxOn",
772                "\\group_begin:",
773                "\\int_zero:N \\l_a",
774                "\\ExplSyntaxOff",
775            ]
776        );
777    }
778
779    #[test]
780    fn npn_definition_is_one_unit() {
781        let got =
782            statements("\\ExplSyntaxOn\n\\cs_new:Npn \\foo:n #1\n  { body #1 }\n\\ExplSyntaxOff\n");
783        assert_eq!(
784            got,
785            vec![
786                "\\ExplSyntaxOn",
787                "\\cs_new:Npn \\foo:n #1 { body #1 }",
788                "\\ExplSyntaxOff",
789            ]
790        );
791    }
792
793    #[test]
794    fn peel_back_reclaims_over_attached_group() {
795        let got = statements("\\ExplSyntaxOn\n\\cs_new:Nn \\foo:n\n  { body }\n\\ExplSyntaxOff\n");
796        assert_eq!(
797            got,
798            vec![
799                "\\ExplSyntaxOn",
800                "\\cs_new:Nn \\foo:n { body }",
801                "\\ExplSyntaxOff",
802            ]
803        );
804    }
805
806    #[test]
807    fn exp_args_chain_is_one_unit() {
808        let got = statements(
809            "\\ExplSyntaxOn\n\\exp_args:NNo \\tl_set:Nn \\l_a { \\l_b }\n\\ExplSyntaxOff\n",
810        );
811        assert_eq!(
812            got,
813            vec![
814                "\\ExplSyntaxOn",
815                "\\exp_args:NNo \\tl_set:Nn \\l_a { \\l_b }",
816                "\\ExplSyntaxOff",
817            ]
818        );
819    }
820
821    #[test]
822    fn hash_parameter_satisfies_single_token_slot() {
823        let got = statements("\\ExplSyntaxOn\n\\tl_set:Nn #1 { x }\n\\ExplSyntaxOff\n");
824        assert_eq!(
825            got,
826            vec!["\\ExplSyntaxOn", "\\tl_set:Nn #1 { x }", "\\ExplSyntaxOff"]
827        );
828    }
829
830    #[test]
831    fn relation_character_satisfies_single_token_slot() {
832        let got = statements(
833            "\\ExplSyntaxOn\n\\int_compare:nNnTF { \\l_a } = { 1 } { yes } { no } \\foo:\n\\ExplSyntaxOff\n",
834        );
835        assert_eq!(
836            got,
837            vec![
838                "\\ExplSyntaxOn",
839                "\\int_compare:nNnTF { \\l_a } = { 1 } { yes } { no }",
840                "\\foo:",
841                "\\ExplSyntaxOff",
842            ]
843        );
844    }
845
846    #[test]
847    fn relation_character_unit_is_newline_invariant() {
848        let inline = statements(
849            "\\ExplSyntaxOn\n\\int_compare:nNnTF { \\l_a } = { 1 } { yes } { no } \\foo:\n\\ExplSyntaxOff\n",
850        );
851        let broken = statements(
852            "\\ExplSyntaxOn\n\\int_compare:nNnTF { \\l_a } = { 1 }\n  { yes } { no }\n\\foo:\n\\ExplSyntaxOff\n",
853        );
854        assert_eq!(inline, broken);
855    }
856
857    #[test]
858    fn multi_character_word_does_not_satisfy_single_token_slot() {
859        let got = statements(
860            "\\ExplSyntaxOn\n\\int_compare:nNnT { \\l_a } <= { 1 } { yes }\n\\foo:\n\\ExplSyntaxOff\n",
861        );
862        assert_eq!(
863            got,
864            vec![
865                "\\ExplSyntaxOn",
866                "\\int_compare:nNnT { \\l_a } <= { 1 } { yes }",
867                "\\foo:",
868                "\\ExplSyntaxOff",
869            ]
870        );
871    }
872
873    #[test]
874    fn delimited_parameter_text_peels_the_body() {
875        let got = statements(
876            "\\ExplSyntaxOn\n\\cs_new:Npn \\foo:w #1 \\q_stop { body }\n\\ExplSyntaxOff\n",
877        );
878        assert_eq!(
879            got,
880            vec![
881                "\\ExplSyntaxOn",
882                "\\cs_new:Npn \\foo:w #1 \\q_stop { body }",
883                "\\ExplSyntaxOff",
884            ]
885        );
886    }
887
888    #[test]
889    fn comment_in_a_consumed_slot_ends_the_fallback_line() {
890        let got = statements(
891            "\\ExplSyntaxOn\n\\exp_after:wN \\foo \\tl_set:Nn \\l_a\n% doc\n{ x } \\group_begin:\n\\ExplSyntaxOff\n",
892        );
893        assert_eq!(
894            got,
895            vec![
896                "\\ExplSyntaxOn",
897                "\\exp_after:wN \\foo \\tl_set:Nn \\l_a % doc { x }",
898                "\\group_begin:",
899                "\\ExplSyntaxOff",
900            ]
901        );
902    }
903
904    #[test]
905    fn unknown_head_falls_back_to_its_line() {
906        let got = statements(
907            "\\ExplSyntaxOn\n\\exp_after:wN \\foo \\tl_set:Nn \\l_a { x }\n\\group_begin:\n\\ExplSyntaxOff\n",
908        );
909        assert_eq!(
910            got,
911            vec![
912                "\\ExplSyntaxOn",
913                "\\exp_after:wN \\foo \\tl_set:Nn \\l_a { x }",
914                "\\group_begin:",
915                "\\ExplSyntaxOff",
916            ]
917        );
918    }
919
920    #[test]
921    fn shape_mismatch_falls_back() {
922        let got = statements("\\ExplSyntaxOn\n\\tl_set:Nn\n\\l_a\n\\ExplSyntaxOff\n");
923        assert_eq!(
924            got,
925            vec!["\\ExplSyntaxOn", "\\tl_set:Nn", "\\l_a", "\\ExplSyntaxOff"]
926        );
927    }
928
929    #[test]
930    fn trailing_comment_rides_the_statement() {
931        let got = statements("\\ExplSyntaxOn\n\\tl_set:Nn \\l_a { x } % note\n\\ExplSyntaxOff\n");
932        assert_eq!(
933            got,
934            vec![
935                "\\ExplSyntaxOn",
936                "\\tl_set:Nn \\l_a { x } % note",
937                "\\ExplSyntaxOff",
938            ]
939        );
940    }
941
942    #[test]
943    fn leftover_attached_group_rides_the_statement() {
944        let got = statements("\\ExplSyntaxOn\n\\use:n { a } { b }\n\\ExplSyntaxOff\n");
945        assert_eq!(
946            got,
947            vec!["\\ExplSyntaxOn", "\\use:n { a } { b }", "\\ExplSyntaxOff"]
948        );
949    }
950
951    #[test]
952    fn conditional_call_is_one_unit() {
953        let got = statements(
954            "\\ExplSyntaxOn\n\\str_if_eq:nnTF { a } { b }\n  { yes }\n  { no }\n\\ExplSyntaxOff\n",
955        );
956        assert_eq!(
957            got,
958            vec![
959                "\\ExplSyntaxOn",
960                "\\str_if_eq:nnTF { a } { b } { yes } { no }",
961                "\\ExplSyntaxOff",
962            ]
963        );
964    }
965
966    fn branch_texts(src: &str, head: usize) -> Option<Vec<String>> {
967        let parsed = parse(src);
968        assert!(parsed.errors.is_empty(), "test source should parse cleanly");
969        let root = SyntaxNode::new_root(parsed.green);
970        let para = root
971            .children()
972            .find(|n| n.kind() == SyntaxKind::PARAGRAPH)
973            .expect("a paragraph");
974        let elements: Vec<SyntaxElement> = para.children_with_tokens().collect();
975        let unit = expl3_unit(&elements, head)?;
976        Some(
977            unit.branches
978                .iter()
979                .map(|range| normalize(&root.text().slice(*range).to_string()))
980                .collect(),
981        )
982    }
983
984    fn head_of(src: &str, name: &str) -> usize {
985        let parsed = parse(src);
986        let root = SyntaxNode::new_root(parsed.green);
987        let para = root
988            .children()
989            .find(|n| n.kind() == SyntaxKind::PARAGRAPH)
990            .expect("a paragraph");
991        para.children_with_tokens()
992            .position(|el| {
993                el.as_node().is_some_and(|n| {
994                    n.kind() == SyntaxKind::COMMAND
995                        && command_name(n).is_some_and(|got| got == name)
996                })
997            })
998            .unwrap_or_else(|| panic!("no command named {name}"))
999    }
1000
1001    #[test]
1002    fn branches_are_resolved_wherever_attachment_put_them() {
1003        let head_attached = "\\ExplSyntaxOn\n\\tl_if_empty:nTF {#1} { T } { F }\n";
1004        assert_eq!(
1005            branch_texts(head_attached, head_of(head_attached, "tl_if_empty:nTF")),
1006            Some(vec!["{ T }".to_string(), "{ F }".to_string()])
1007        );
1008
1009        let one_sibling = "\\ExplSyntaxOn\n\\seq_if_in:NnTF \\l_seq {item} { T } { F }\n";
1010        assert_eq!(
1011            branch_texts(one_sibling, head_of(one_sibling, "seq_if_in:NnTF")),
1012            Some(vec!["{ T }".to_string(), "{ F }".to_string()])
1013        );
1014
1015        let two_siblings = "\\ExplSyntaxOn\n\\prop_get:NnNTF \\p {k} \\l { T } { F }\n";
1016        assert_eq!(
1017            branch_texts(two_siblings, head_of(two_siblings, "prop_get:NnNTF")),
1018            Some(vec!["{ T }".to_string(), "{ F }".to_string()])
1019        );
1020
1021        let stream_level = "\\ExplSyntaxOn\n\\int_compare:nNnTF {a} = { 1 } { T } { F }\n";
1022        assert_eq!(
1023            branch_texts(stream_level, head_of(stream_level, "int_compare:nNnTF")),
1024            Some(vec!["{ T }".to_string(), "{ F }".to_string()])
1025        );
1026    }
1027
1028    #[test]
1029    fn a_non_conditional_unit_has_no_branches() {
1030        let src = "\\ExplSyntaxOn\n\\tl_set:Nn \\l_a { x }\n";
1031        assert_eq!(branch_texts(src, head_of(src, "tl_set:Nn")), Some(vec![]));
1032    }
1033
1034    #[test]
1035    fn an_underivable_head_resolves_no_unit() {
1036        let src = "\\ExplSyntaxOn\n\\odd_if:wTF \\a \\b { T } { F }\n";
1037        assert_eq!(branch_texts(src, head_of(src, "odd_if:wTF")), None);
1038    }
1039
1040    #[test]
1041    fn a_blank_line_cut_unit_reports_no_branches() {
1042        let src = "\\ExplSyntaxOn\n\\use:n { \\prop_get:NnNTF \\p {k} \\l { T }\n\n{ F } }\n";
1043        let parsed = parse(src);
1044        assert!(parsed.errors.is_empty());
1045        let root = SyntaxNode::new_root(parsed.green);
1046        let group = root
1047            .descendants()
1048            .find(|n| n.kind() == SyntaxKind::GROUP)
1049            .expect("a group");
1050        let body: Vec<SyntaxElement> = group
1051            .children_with_tokens()
1052            .filter(|el| !matches!(el.kind(), SyntaxKind::L_BRACE | SyntaxKind::R_BRACE))
1053            .collect();
1054        let head = body
1055            .iter()
1056            .position(|el| el.as_node().is_some())
1057            .expect("the head command");
1058        let unit = expl3_unit(&body, head).expect("the partial unit still resolves");
1059        assert_eq!(unit.branches, vec![]);
1060    }
1061
1062    #[test]
1063    fn blank_line_ends_the_unit() {
1064        let src = "\\ExplSyntaxOn\n\\use:n { \\tl_set:Nn \\l_a\n\n  { x } }\n\\ExplSyntaxOff\n";
1065        let parsed = parse(src);
1066        assert!(parsed.errors.is_empty());
1067        let root = SyntaxNode::new_root(parsed.green);
1068        let group = root
1069            .descendants()
1070            .find(|n| n.kind() == SyntaxKind::GROUP)
1071            .expect("a group");
1072        let body: Vec<SyntaxElement> = group
1073            .children_with_tokens()
1074            .filter(|el| !matches!(el.kind(), SyntaxKind::L_BRACE | SyntaxKind::R_BRACE))
1075            .collect();
1076        assert_eq!(statement_texts(&body), vec!["\\tl_set:Nn \\l_a", "{ x }"]);
1077    }
1078
1079    #[test]
1080    fn guard_mid_unit_aborts_to_fallback() {
1081        use crate::parser::lexer::LexConfig;
1082        use crate::parser::{LatexFlavor, parse_with_flavor};
1083        let src = "% \\begin{macrocode}\n\\ExplSyntaxOn\n\\tl_set:Nn \\l_a\n%<latexrelease>  { x }\n\\ExplSyntaxOff\n% \\end{macrocode}\n";
1084        let config = LexConfig {
1085            flavor: LatexFlavor::Package,
1086            dtx: true,
1087        };
1088        let parsed = parse_with_flavor(src, config);
1089        assert!(parsed.errors.is_empty(), "test source should parse cleanly");
1090        let root = SyntaxNode::new_root(parsed.green);
1091        let para = root
1092            .descendants()
1093            .find(|n| n.kind() == SyntaxKind::PARAGRAPH)
1094            .expect("a paragraph");
1095        let elements: Vec<SyntaxElement> = para.children_with_tokens().collect();
1096        let map = segment_expl_statements(&elements);
1097        assert_eq!(
1098            statement_texts(&elements),
1099            vec![
1100                "\\ExplSyntaxOn",
1101                "\\tl_set:Nn \\l_a %<latexrelease> { x }",
1102                "\\ExplSyntaxOff",
1103            ]
1104        );
1105        let guarded_end = elements
1106            .iter()
1107            .position(|el| el.to_string().contains("latexrelease"))
1108            .expect("the guarded sibling");
1109        assert!(
1110            map.is_fallback(guarded_end),
1111            "the aborted unit must be a fallback statement"
1112        );
1113    }
1114
1115    #[test]
1116    fn e_and_f_letters_consume_braced_groups() {
1117        let got = statements(
1118            "\\ExplSyntaxOn\n\\tl_set:Ne \\l_a\n  { x }\n\\tl_set:Nf \\l_b\n  { y }\n\\ExplSyntaxOff\n",
1119        );
1120        assert_eq!(
1121            got,
1122            vec![
1123                "\\ExplSyntaxOn",
1124                "\\tl_set:Ne \\l_a { x }",
1125                "\\tl_set:Nf \\l_b { y }",
1126                "\\ExplSyntaxOff",
1127            ]
1128        );
1129    }
1130
1131    #[test]
1132    fn stream_ending_mid_unit_falls_back() {
1133        let src = "\\ExplSyntaxOn\n\\use:n { \\tl_set:Nn \\l_a }\n\\ExplSyntaxOff\n";
1134        let parsed = parse(src);
1135        assert!(parsed.errors.is_empty());
1136        let root = SyntaxNode::new_root(parsed.green);
1137        let group = root
1138            .descendants()
1139            .find(|n| n.kind() == SyntaxKind::GROUP)
1140            .expect("a group");
1141        let body: Vec<SyntaxElement> = group
1142            .children_with_tokens()
1143            .filter(|el| !matches!(el.kind(), SyntaxKind::L_BRACE | SyntaxKind::R_BRACE))
1144            .collect();
1145        let map = segment_expl_statements(&body);
1146        assert_eq!(statement_texts(&body), vec!["\\tl_set:Nn \\l_a"]);
1147        let head = body
1148            .iter()
1149            .position(|el| el.as_node().is_some())
1150            .expect("the head command");
1151        assert!(
1152            map.is_fallback(head),
1153            "a unit cut off by the stream end must be a fallback statement"
1154        );
1155    }
1156
1157    #[test]
1158    fn a_multi_line_group_node_does_not_end_a_fallback_line() {
1159        let src = "\\ExplSyntaxOn\n\
1160                   \\int_do_until:w { \\l_tmpa_int } > {#2}\n\
1161                   { \\lipsum_add:V { \\l_tmpa_int }\n\
1162                   \\int_incr:N \\l_tmpa_int } \\tl_put_right:NV \\l_a \\l_b\n\
1163                   \\ExplSyntaxOff\n";
1164        let parsed = parse(src);
1165        assert!(parsed.errors.is_empty());
1166        let root = SyntaxNode::new_root(parsed.green);
1167        let elements: Vec<SyntaxElement> = root
1168            .first_child()
1169            .expect("the paragraph")
1170            .children_with_tokens()
1171            .collect();
1172        let map = segment_expl_statements(&elements);
1173
1174        assert_eq!(
1175            statement_texts(&elements),
1176            vec![
1177                "\\ExplSyntaxOn",
1178                "\\int_do_until:w { \\l_tmpa_int } > {#2}",
1179                "{ \\lipsum_add:V { \\l_tmpa_int } \\int_incr:N \\l_tmpa_int } \
1180                 \\tl_put_right:NV \\l_a \\l_b",
1181                "\\ExplSyntaxOff",
1182            ]
1183        );
1184
1185        let group = elements
1186            .iter()
1187            .position(|el| el.kind() == SyntaxKind::GROUP && el.to_string().contains('\n'))
1188            .expect("the multi-line group");
1189        assert!(
1190            map.is_fallback(group),
1191            "the group belongs to a fallback statement"
1192        );
1193        assert!(
1194            !map.boundary_after(group),
1195            "a multi-line group's own newlines must not end the fallback line"
1196        );
1197
1198        let head = elements
1199            .iter()
1200            .skip(group)
1201            .position(|el| {
1202                el.as_node()
1203                    .is_some_and(|n| n.kind() == SyntaxKind::COMMAND)
1204            })
1205            .map(|off| group + off)
1206            .expect("the trailing recognized head");
1207        assert!(
1208            map.glue_before(head),
1209            "a recognized head mid-fallback-line owes an unbreakable gap"
1210        );
1211    }
1212
1213    #[test]
1214    fn own_line_comment_in_attached_span_rides_the_sibling() {
1215        let got =
1216            statements("\\ExplSyntaxOn\n\\tl_set:Nn \\l_a\n% note\n  { x }\n\\ExplSyntaxOff\n");
1217        assert_eq!(
1218            got,
1219            vec![
1220                "\\ExplSyntaxOn",
1221                "\\tl_set:Nn \\l_a % note { x }",
1222                "\\ExplSyntaxOff",
1223            ]
1224        );
1225    }
1226
1227    #[test]
1228    fn own_line_comment_at_sibling_level_ends_the_unit() {
1229        let got = statements(
1230            "\\ExplSyntaxOn\n\\cs_new:Npn \\foo:n\n% note\n#1 { body }\n\\ExplSyntaxOff\n",
1231        );
1232        assert_eq!(
1233            got,
1234            vec![
1235                "\\ExplSyntaxOn",
1236                "\\cs_new:Npn \\foo:n",
1237                "% note",
1238                "#1 { body }",
1239                "\\ExplSyntaxOff",
1240            ]
1241        );
1242    }
1243}