1pub mod calls;
42pub mod mode;
43pub mod symbols;
44pub mod variants;
45
46use std::collections::VecDeque;
47
48use rowan::TextRange;
49
50use crate::ast::command_name;
51use crate::parser::lexer::expl_toggle;
52use crate::syntax::{SyntaxElement, SyntaxKind, SyntaxNode, is_collapsible_trivia, is_param_digit};
53
54#[derive(Debug, Clone, Copy, PartialEq, Eq)]
56pub enum Expl3Slot {
57 SingleToken,
59 Group,
61 Branch,
64 ParameterText,
67}
68
69pub fn expl3_slots(name: &str) -> Option<Vec<Expl3Slot>> {
79 let argspec = name.rsplit_once(':')?.1;
80 let chars: Vec<char> = argspec.chars().collect();
81 let branches = chars
82 .iter()
83 .rev()
84 .take_while(|c| matches!(c, 'T' | 'F'))
85 .count();
86 let mut slots = Vec::with_capacity(chars.len());
87 for c in &chars[..chars.len() - branches] {
88 slots.push(match c {
91 'N' | 'V' => Expl3Slot::SingleToken,
92 'n' | 'c' | 'v' | 'o' | 'x' | 'e' | 'f' => Expl3Slot::Group,
93 'p' => Expl3Slot::ParameterText,
94 _ => return None,
95 });
96 }
97 slots.extend(std::iter::repeat_n(Expl3Slot::Branch, branches));
98 Some(slots)
99}
100
101pub fn conditional_branches(name: &str) -> Option<usize> {
115 let argspec = name.rsplit_once(':')?.1;
116 let n = argspec
117 .chars()
118 .rev()
119 .take_while(|c| *c == 'T' || *c == 'F')
120 .count();
121 (n > 0).then_some(n)
122}
123
124#[cfg(test)]
125mod tests {
126 use super::*;
127 use Expl3Slot::*;
128
129 #[test]
130 fn slots_read_from_name_suffix() {
131 assert_eq!(
132 expl3_slots("cs_new:Npn"),
133 Some(vec![SingleToken, ParameterText, Group])
134 );
135 assert_eq!(
136 expl3_slots("str_if_eq:nnTF"),
137 Some(vec![Group, Group, Branch, Branch])
138 );
139 assert_eq!(
140 expl3_slots("prop_get:NnNTF"),
141 Some(vec![SingleToken, Group, SingleToken, Branch, Branch])
142 );
143 assert_eq!(expl3_slots("tl_set:Nn"), Some(vec![SingleToken, Group]));
144 assert_eq!(
145 expl3_slots("exp_args:NNo"),
146 Some(vec![SingleToken, SingleToken, Group])
147 );
148 assert_eq!(expl3_slots("tl_set:Nv"), Some(vec![SingleToken, Group]));
149 assert_eq!(expl3_slots("use:c"), Some(vec![Group]));
150 assert_eq!(expl3_slots("tl_set:Nx"), Some(vec![SingleToken, Group]));
151 }
152
153 #[test]
154 fn zero_argument_names_are_recognized() {
155 assert_eq!(expl3_slots("scan_stop:"), Some(vec![]));
156 assert_eq!(expl3_slots("group_begin:"), Some(vec![]));
157 assert_eq!(expl3_slots("prg_return_true:"), Some(vec![]));
158 }
159
160 #[test]
161 fn underivable_specs_are_unrecognized() {
162 assert_eq!(expl3_slots("use_none_delimit_by_q_stop:w"), None);
163 assert_eq!(expl3_slots("exp_after:wN"), None);
164 assert_eq!(expl3_slots("tex_relax:D"), None);
165 assert_eq!(expl3_slots("odd:TnF"), None);
166 assert_eq!(expl3_slots("odd:nZn"), None);
167 }
168
169 #[test]
170 fn colonless_names_are_unrecognized() {
171 assert_eq!(expl3_slots("def"), None);
172 assert_eq!(expl3_slots("@ifpackageloaded"), None);
173 assert_eq!(expl3_slots("IfBooleanTF"), None);
174 assert_eq!(expl3_slots("l_tmpa_tl"), None);
175 }
176
177 #[test]
178 fn exp_internal_drivers() {
179 assert_eq!(expl3_slots("::n"), Some(vec![Group]));
180 assert_eq!(expl3_slots(":::"), Some(vec![]));
181 }
182
183 #[test]
184 fn conditional_branches_read_from_name_suffix() {
185 assert_eq!(conditional_branches("tl_if_empty:nTF"), Some(2));
186 assert_eq!(conditional_branches("bool_if:nT"), Some(1));
187 assert_eq!(conditional_branches("bool_if:nF"), Some(1));
188 assert_eq!(conditional_branches("str_if_eq:nnTF"), Some(2));
189 assert_eq!(conditional_branches("int_compare:nNnTF"), Some(2));
190 assert_eq!(conditional_branches("seq_map_inline:Nn"), None);
191 assert_eq!(conditional_branches("prg_return_true:"), None);
192 assert_eq!(conditional_branches("tl_new:N"), None);
193 assert_eq!(conditional_branches("@ifpackageloaded"), None);
194 assert_eq!(conditional_branches("IfBooleanTF"), None);
195 }
196
197 #[test]
198 fn branches_survive_underivable_arity() {
199 assert_eq!(expl3_slots("odd_if:wTF"), None);
200 assert_eq!(conditional_branches("odd_if:wTF"), Some(2));
201 }
202}
203
204pub struct StatementMap {
206 flags: Vec<ElementFlags>,
207}
208
209#[derive(Clone, Copy, Default)]
210struct ElementFlags(u8);
211
212impl ElementFlags {
213 const BOUNDARY_AFTER: u8 = 1 << 0;
214 const GLUE_BEFORE: u8 = 1 << 1;
215 const GLUED: u8 = 1 << 2;
216 const FALLBACK: u8 = 1 << 3;
217
218 fn contains(self, flag: u8) -> bool {
219 self.0 & flag != 0
220 }
221
222 fn insert(&mut self, flag: u8) {
223 self.0 |= flag;
224 }
225}
226
227impl StatementMap {
228 pub fn boundary_after(&self, idx: usize) -> bool {
229 self.flags
230 .get(idx)
231 .is_some_and(|flags| flags.contains(ElementFlags::BOUNDARY_AFTER))
232 }
233
234 pub fn glue_before(&self, idx: usize) -> bool {
235 self.flags
236 .get(idx)
237 .is_some_and(|flags| flags.contains(ElementFlags::GLUE_BEFORE))
238 }
239
240 pub fn is_glued(&self, idx: usize) -> bool {
241 self.flags
242 .get(idx)
243 .is_some_and(|flags| flags.contains(ElementFlags::GLUED))
244 }
245
246 pub fn is_fallback(&self, idx: usize) -> bool {
247 self.flags
248 .get(idx)
249 .is_some_and(|flags| flags.contains(ElementFlags::FALLBACK))
250 }
251}
252
253pub fn segment_expl_statements(elements: &[SyntaxElement]) -> StatementMap {
255 let mut flags = vec![ElementFlags::default(); elements.len()];
256 let mut i = 0;
257 while i < elements.len() {
258 match &elements[i] {
259 SyntaxElement::Token(t) if is_collapsible_trivia(t.kind()) => i += 1,
260 SyntaxElement::Token(t)
261 if matches!(
262 t.kind(),
263 SyntaxKind::COMMENT | SyntaxKind::GUARD | SyntaxKind::DOC_MARGIN
264 ) =>
265 {
266 if followed_by_newline(elements, i) {
267 flags[i].insert(ElementFlags::BOUNDARY_AFTER);
268 }
269 i += 1;
270 }
271 SyntaxElement::Node(n) if n.kind() == SyntaxKind::COMMAND => {
272 match expl3_unit(elements, i) {
273 Some(unit) => {
274 let end = unit.last;
275 let full = absorb_trailing_junk(elements, end);
276 if full > end {
277 for flags in &mut flags[i..=full] {
278 flags.insert(ElementFlags::GLUED);
279 }
280 }
281 flags[full].insert(ElementFlags::BOUNDARY_AFTER);
282 i = full + 1;
283 }
284 None => i = fallback_line(elements, i, &mut flags),
285 }
286 }
287 _ => i = fallback_line(elements, i, &mut flags),
288 }
289 }
290 StatementMap { flags }
291}
292
293fn node_is_expl_toggle(node: &SyntaxNode) -> bool {
294 node.children_with_tokens()
295 .filter_map(|el| el.into_token())
296 .find(|t| t.kind() == SyntaxKind::CONTROL_WORD)
297 .is_some_and(|t| expl_toggle(t.text()).is_some())
298}
299
300fn is_recognized_head(node: &SyntaxNode) -> bool {
301 node.kind() == SyntaxKind::COMMAND
302 && (node_is_expl_toggle(node)
303 || command_name(node).is_some_and(|name| expl3_slots(&name).is_some()))
304}
305
306fn followed_by_newline(elements: &[SyntaxElement], idx: usize) -> bool {
307 for element in &elements[idx + 1..] {
308 match element {
309 SyntaxElement::Token(t) if t.kind() == SyntaxKind::WHITESPACE => {}
310 SyntaxElement::Token(t) if t.kind() == SyntaxKind::NEWLINE => return true,
311 _ => return false,
312 }
313 }
314 true
315}
316
317fn fallback_line(elements: &[SyntaxElement], start: usize, flags: &mut [ElementFlags]) -> usize {
318 let mut last = start;
319 let mut j = start;
320 while j < elements.len() {
321 match &elements[j] {
322 SyntaxElement::Token(t) if is_collapsible_trivia(t.kind()) => {
323 if t.kind() == SyntaxKind::NEWLINE {
324 flags[last].insert(ElementFlags::BOUNDARY_AFTER);
325 for flags in &mut flags[start..=last] {
326 flags.insert(ElementFlags::FALLBACK);
327 }
328 return j;
329 }
330 j += 1;
331 }
332 element => {
333 if j > start
334 && let SyntaxElement::Node(n) = element
335 && is_recognized_head(n)
336 {
337 flags[j].insert(ElementFlags::GLUE_BEFORE);
338 }
339 last = j;
340 j += 1;
341 if let SyntaxElement::Node(n) = element
342 && n.kind() == SyntaxKind::COMMAND
343 && node_carries_bare_line_break(n)
344 {
345 flags[last].insert(ElementFlags::BOUNDARY_AFTER);
346 for flags in &mut flags[start..=last] {
347 flags.insert(ElementFlags::FALLBACK);
348 }
349 return j;
350 }
351 }
352 }
353 }
354 flags[last].insert(ElementFlags::BOUNDARY_AFTER);
355 for flags in &mut flags[start..=last] {
356 flags.insert(ElementFlags::FALLBACK);
357 }
358 elements.len()
359}
360
361fn node_carries_bare_line_break(node: &SyntaxNode) -> bool {
362 let mut after_newline = false;
363 for child in node.children_with_tokens() {
364 match &child {
365 SyntaxElement::Token(t) if t.kind() == SyntaxKind::NEWLINE => after_newline = true,
366 SyntaxElement::Token(t) if is_collapsible_trivia(t.kind()) => {}
367 SyntaxElement::Node(n)
368 if matches!(n.kind(), SyntaxKind::GROUP | SyntaxKind::OPTIONAL) =>
369 {
370 after_newline = false;
371 }
372 _ => {
373 if after_newline {
374 return true;
375 }
376 }
377 }
378 }
379 false
380}
381
382fn absorb_trailing_junk(elements: &[SyntaxElement], end: usize) -> usize {
383 let mut end = end;
384 let mut j = end + 1;
385 while j < elements.len() {
386 match &elements[j] {
387 SyntaxElement::Token(t) if is_collapsible_trivia(t.kind()) => {
388 if t.kind() == SyntaxKind::NEWLINE {
389 break;
390 }
391 j += 1;
392 }
393 SyntaxElement::Token(t) if t.kind() == SyntaxKind::COMMENT => {
394 end = j;
395 break;
396 }
397 SyntaxElement::Token(t)
398 if matches!(t.kind(), SyntaxKind::GUARD | SyntaxKind::DOC_MARGIN) =>
399 {
400 break;
401 }
402 SyntaxElement::Node(n) if n.kind() == SyntaxKind::GROUP => break,
403 SyntaxElement::Node(n) if is_recognized_head(n) => {
404 break;
405 }
406 _ => {
407 end = j;
408 j += 1;
409 }
410 }
411 }
412 end
413}
414
415enum Stop {
416 End,
417 Abort,
418}
419
420fn consume_unit(
421 elements: &[SyntaxElement],
422 head_idx: usize,
423 slots: &[Expl3Slot],
424) -> Option<Expl3Unit> {
425 let head = elements[head_idx].as_node()?;
426 let mut cur = UnitCursor::new(elements, head_idx, head);
427 let mut branches = Vec::new();
428 let mut complete = true;
429 for slot in slots {
430 let took = match slot {
431 Expl3Slot::SingleToken => cur.take_single_token(),
432 Expl3Slot::Group => cur.take_group().map(|_| ()),
433 Expl3Slot::Branch => cur.take_group().map(|el| branches.push(el.text_range())),
434 Expl3Slot::ParameterText => cur.take_parameter_text(),
435 };
436 match took {
437 Ok(()) => {}
438 Err(Stop::End) => {
439 complete = false;
440 break;
441 }
442 Err(Stop::Abort) => return None,
443 }
444 }
445 cur.extend_over_attachable_tail();
446 Some(Expl3Unit {
447 last: cur.last_sib,
448 branches: if complete { branches } else { Vec::new() },
449 })
450}
451
452#[derive(Debug, Clone, PartialEq, Eq)]
453pub struct Expl3Unit {
455 pub last: usize,
456 pub branches: Vec<TextRange>,
457}
458
459pub fn expl3_unit(elements: &[SyntaxElement], head_idx: usize) -> Option<Expl3Unit> {
461 let node = elements.get(head_idx)?.as_node()?;
462 if !is_recognized_head(node) {
463 return None;
464 }
465 let slots = if node_is_expl_toggle(node) {
466 Vec::new()
467 } else {
468 expl3_slots(&command_name(node)?)?
469 };
470 consume_unit(elements, head_idx, &slots)
471}
472
473struct UnitCursor<'a> {
474 elements: &'a [SyntaxElement],
475 queue: VecDeque<SyntaxElement>,
476 sib: usize,
477 last_sib: usize,
478 peeked: Option<(SyntaxElement, Option<usize>)>,
479 chain: bool,
480}
481
482impl<'a> UnitCursor<'a> {
483 fn new(elements: &'a [SyntaxElement], head_idx: usize, head: &SyntaxNode) -> Self {
484 let mut cur = UnitCursor {
485 elements,
486 queue: VecDeque::new(),
487 sib: head_idx + 1,
488 last_sib: head_idx,
489 peeked: None,
490 chain: true,
491 };
492 cur.queue_children_after_name(head, false);
493 cur
494 }
495
496 fn queue_children_after_name(&mut self, node: &SyntaxNode, front: bool) {
497 let mut seen_name = false;
498 let mut after: Vec<SyntaxElement> = Vec::new();
499 for child in node.children_with_tokens() {
500 if seen_name {
501 after.push(child);
502 } else if matches!(
503 child.kind(),
504 SyntaxKind::CONTROL_WORD | SyntaxKind::CONTROL_SYMBOL
505 ) {
506 seen_name = true;
507 }
508 }
509 if front {
510 for el in after.into_iter().rev() {
511 self.queue.push_front(el);
512 }
513 } else {
514 self.queue.extend(after);
515 }
516 }
517
518 fn peek(&mut self) -> Result<&SyntaxElement, Stop> {
519 if self.peeked.is_none() {
520 self.peeked = Some(self.advance()?);
521 }
522 Ok(&self.peeked.as_ref().expect("just filled").0)
523 }
524
525 fn bump(&mut self) -> Result<SyntaxElement, Stop> {
526 let (el, sib_idx) = match self.peeked.take() {
527 Some(peeked) => peeked,
528 None => self.advance()?,
529 };
530 if let Some(idx) = sib_idx {
531 self.last_sib = idx;
532 }
533 Ok(el)
534 }
535
536 fn advance(&mut self) -> Result<(SyntaxElement, Option<usize>), Stop> {
537 let mut gap_newlines = 0usize;
538 loop {
539 let (el, sib_idx) = if let Some(el) = self.queue.pop_front() {
540 (el, None)
541 } else {
542 let Some(el) = self.elements.get(self.sib) else {
543 return Err(Stop::Abort);
544 };
545 if let SyntaxElement::Token(t) = el
546 && t.kind() == SyntaxKind::NEWLINE
547 && gap_newlines >= 1
548 {
549 return Err(Stop::End);
550 }
551 let idx = self.sib;
552 self.sib += 1;
553 (el.clone(), Some(idx))
554 };
555 match &el {
556 SyntaxElement::Token(t) if is_collapsible_trivia(t.kind()) => {
557 if t.kind() == SyntaxKind::NEWLINE {
558 gap_newlines += 1;
559 if gap_newlines >= 2 {
560 return Err(Stop::End);
561 }
562 }
563 }
564 SyntaxElement::Token(t) if t.kind() == SyntaxKind::COMMENT => {}
565 SyntaxElement::Token(t) if t.kind() == SyntaxKind::TILDE => {}
566 SyntaxElement::Token(t)
567 if matches!(t.kind(), SyntaxKind::GUARD | SyntaxKind::DOC_MARGIN) =>
568 {
569 return Err(Stop::Abort);
570 }
571 _ => return Ok((el, sib_idx)),
572 }
573 }
574 }
575
576 fn take_single_token(&mut self) -> Result<(), Stop> {
577 let el = self.bump()?;
578 match &el {
579 SyntaxElement::Token(t)
580 if matches!(
581 t.kind(),
582 SyntaxKind::CONTROL_WORD | SyntaxKind::CONTROL_SYMBOL
583 ) =>
584 {
585 self.chain = false;
586 Ok(())
587 }
588 SyntaxElement::Token(t)
589 if t.kind() == SyntaxKind::WORD && t.text().chars().count() == 1 =>
590 {
591 self.chain = false;
592 Ok(())
593 }
594 SyntaxElement::Token(t) if t.kind() == SyntaxKind::HASH => {
595 self.chain = false;
596 loop {
597 let next = self.bump()?;
598 match &next {
599 SyntaxElement::Token(t) if t.kind() == SyntaxKind::HASH => {}
600 SyntaxElement::Token(t)
601 if t.kind() == SyntaxKind::WORD && is_param_digit(t) =>
602 {
603 return Ok(());
604 }
605 _ => return Err(Stop::Abort),
606 }
607 }
608 }
609 SyntaxElement::Node(n) if n.kind() == SyntaxKind::COMMAND => {
610 self.queue_children_after_name(n, true);
611 self.chain = true;
612 Ok(())
613 }
614 SyntaxElement::Node(n) if n.kind() == SyntaxKind::GROUP => Ok(()),
615 _ => Err(Stop::Abort),
616 }
617 }
618
619 fn take_group(&mut self) -> Result<SyntaxElement, Stop> {
620 let el = self.bump()?;
621 match &el {
622 SyntaxElement::Node(n) if n.kind() == SyntaxKind::GROUP => Ok(el),
623 _ => Err(Stop::Abort),
624 }
625 }
626
627 fn take_parameter_text(&mut self) -> Result<(), Stop> {
628 loop {
629 if let SyntaxElement::Node(n) = self.peek()?
630 && n.kind() == SyntaxKind::GROUP
631 {
632 return Ok(());
633 }
634 let el = self.bump()?;
635 match &el {
636 SyntaxElement::Token(_) => self.chain = false,
637 SyntaxElement::Node(n) if n.kind() == SyntaxKind::COMMAND => {
638 self.queue_children_after_name(n, true);
639 self.chain = true;
640 }
641 SyntaxElement::Node(n) if n.kind() == SyntaxKind::OPTIONAL => {}
642 _ => return Err(Stop::Abort),
643 }
644 }
645 }
646
647 fn extend_over_attachable_tail(&mut self) {
648 if let Some((el, sib_idx)) = self.peeked.take()
649 && sib_idx.is_none()
650 {
651 self.update_chain(&el);
652 }
653 while let Some(el) = self.queue.pop_front() {
654 self.update_chain(&el);
655 }
656 if !self.chain {
657 return;
658 }
659 let mut newlines = 0usize;
660 let mut i = self.last_sib + 1;
661 while let Some(el) = self.elements.get(i) {
662 match el {
663 SyntaxElement::Token(t) => match t.kind() {
664 SyntaxKind::NEWLINE => {
665 newlines += 1;
666 if newlines >= 2 {
667 return;
668 }
669 }
670 SyntaxKind::COMMENT => newlines = 0,
671 SyntaxKind::WHITESPACE | SyntaxKind::GUARD | SyntaxKind::DOC_MARGIN => {}
672 _ => return,
673 },
674 SyntaxElement::Node(n)
675 if matches!(n.kind(), SyntaxKind::GROUP | SyntaxKind::OPTIONAL) =>
676 {
677 self.last_sib = i;
678 newlines = 0;
679 }
680 SyntaxElement::Node(_) => return,
681 }
682 i += 1;
683 }
684 }
685
686 fn update_chain(&mut self, el: &SyntaxElement) {
687 match el {
688 SyntaxElement::Node(n) if n.kind() == SyntaxKind::COMMAND => self.chain = true,
689 SyntaxElement::Node(n)
690 if matches!(n.kind(), SyntaxKind::GROUP | SyntaxKind::OPTIONAL) => {}
691 SyntaxElement::Token(t)
692 if is_collapsible_trivia(t.kind())
693 || matches!(
694 t.kind(),
695 SyntaxKind::COMMENT
696 | SyntaxKind::TILDE
697 | SyntaxKind::GUARD
698 | SyntaxKind::DOC_MARGIN
699 ) => {}
700 _ => self.chain = false,
701 }
702 }
703}
704
705#[cfg(test)]
706mod segmentation_tests {
707 use super::*;
708 use crate::parser::parse;
709 use crate::syntax::SyntaxNode;
710
711 fn statements(src: &str) -> Vec<String> {
712 let parsed = parse(src);
713 assert!(parsed.errors.is_empty(), "test source should parse cleanly");
714 let root = SyntaxNode::new_root(parsed.green);
715 let para = root
716 .children()
717 .find(|n| n.kind() == SyntaxKind::PARAGRAPH)
718 .expect("a paragraph");
719 let elements: Vec<SyntaxElement> = para.children_with_tokens().collect();
720 statement_texts(&elements)
721 }
722
723 fn statement_texts(elements: &[SyntaxElement]) -> Vec<String> {
724 let map = segment_expl_statements(elements);
725 let mut out = Vec::new();
726 let mut cur = String::new();
727 for (i, el) in elements.iter().enumerate() {
728 cur.push_str(&el.to_string());
729 if map.boundary_after(i) {
730 let text = normalize(&cur);
731 if !text.is_empty() {
732 out.push(text);
733 }
734 cur.clear();
735 }
736 }
737 let tail = normalize(&cur);
738 if !tail.is_empty() {
739 out.push(tail);
740 }
741 out
742 }
743
744 fn normalize(s: &str) -> String {
745 s.split_whitespace().collect::<Vec<_>>().join(" ")
746 }
747
748 #[test]
749 fn statements_are_structural_units() {
750 let got = statements(
751 "\\ExplSyntaxOn\n\\tl_set:Nn \\l_a\n { x }\n\\group_begin:\n\\ExplSyntaxOff\n",
752 );
753 assert_eq!(
754 got,
755 vec![
756 "\\ExplSyntaxOn",
757 "\\tl_set:Nn \\l_a { x }",
758 "\\group_begin:",
759 "\\ExplSyntaxOff",
760 ]
761 );
762 }
763
764 #[test]
765 fn same_line_calls_split() {
766 let got =
767 statements("\\ExplSyntaxOn\n\\group_begin: \\int_zero:N \\l_a\n\\ExplSyntaxOff\n");
768 assert_eq!(
769 got,
770 vec![
771 "\\ExplSyntaxOn",
772 "\\group_begin:",
773 "\\int_zero:N \\l_a",
774 "\\ExplSyntaxOff",
775 ]
776 );
777 }
778
779 #[test]
780 fn npn_definition_is_one_unit() {
781 let got =
782 statements("\\ExplSyntaxOn\n\\cs_new:Npn \\foo:n #1\n { body #1 }\n\\ExplSyntaxOff\n");
783 assert_eq!(
784 got,
785 vec![
786 "\\ExplSyntaxOn",
787 "\\cs_new:Npn \\foo:n #1 { body #1 }",
788 "\\ExplSyntaxOff",
789 ]
790 );
791 }
792
793 #[test]
794 fn peel_back_reclaims_over_attached_group() {
795 let got = statements("\\ExplSyntaxOn\n\\cs_new:Nn \\foo:n\n { body }\n\\ExplSyntaxOff\n");
796 assert_eq!(
797 got,
798 vec![
799 "\\ExplSyntaxOn",
800 "\\cs_new:Nn \\foo:n { body }",
801 "\\ExplSyntaxOff",
802 ]
803 );
804 }
805
806 #[test]
807 fn exp_args_chain_is_one_unit() {
808 let got = statements(
809 "\\ExplSyntaxOn\n\\exp_args:NNo \\tl_set:Nn \\l_a { \\l_b }\n\\ExplSyntaxOff\n",
810 );
811 assert_eq!(
812 got,
813 vec![
814 "\\ExplSyntaxOn",
815 "\\exp_args:NNo \\tl_set:Nn \\l_a { \\l_b }",
816 "\\ExplSyntaxOff",
817 ]
818 );
819 }
820
821 #[test]
822 fn hash_parameter_satisfies_single_token_slot() {
823 let got = statements("\\ExplSyntaxOn\n\\tl_set:Nn #1 { x }\n\\ExplSyntaxOff\n");
824 assert_eq!(
825 got,
826 vec!["\\ExplSyntaxOn", "\\tl_set:Nn #1 { x }", "\\ExplSyntaxOff"]
827 );
828 }
829
830 #[test]
831 fn relation_character_satisfies_single_token_slot() {
832 let got = statements(
833 "\\ExplSyntaxOn\n\\int_compare:nNnTF { \\l_a } = { 1 } { yes } { no } \\foo:\n\\ExplSyntaxOff\n",
834 );
835 assert_eq!(
836 got,
837 vec![
838 "\\ExplSyntaxOn",
839 "\\int_compare:nNnTF { \\l_a } = { 1 } { yes } { no }",
840 "\\foo:",
841 "\\ExplSyntaxOff",
842 ]
843 );
844 }
845
846 #[test]
847 fn relation_character_unit_is_newline_invariant() {
848 let inline = statements(
849 "\\ExplSyntaxOn\n\\int_compare:nNnTF { \\l_a } = { 1 } { yes } { no } \\foo:\n\\ExplSyntaxOff\n",
850 );
851 let broken = statements(
852 "\\ExplSyntaxOn\n\\int_compare:nNnTF { \\l_a } = { 1 }\n { yes } { no }\n\\foo:\n\\ExplSyntaxOff\n",
853 );
854 assert_eq!(inline, broken);
855 }
856
857 #[test]
858 fn multi_character_word_does_not_satisfy_single_token_slot() {
859 let got = statements(
860 "\\ExplSyntaxOn\n\\int_compare:nNnT { \\l_a } <= { 1 } { yes }\n\\foo:\n\\ExplSyntaxOff\n",
861 );
862 assert_eq!(
863 got,
864 vec![
865 "\\ExplSyntaxOn",
866 "\\int_compare:nNnT { \\l_a } <= { 1 } { yes }",
867 "\\foo:",
868 "\\ExplSyntaxOff",
869 ]
870 );
871 }
872
873 #[test]
874 fn delimited_parameter_text_peels_the_body() {
875 let got = statements(
876 "\\ExplSyntaxOn\n\\cs_new:Npn \\foo:w #1 \\q_stop { body }\n\\ExplSyntaxOff\n",
877 );
878 assert_eq!(
879 got,
880 vec![
881 "\\ExplSyntaxOn",
882 "\\cs_new:Npn \\foo:w #1 \\q_stop { body }",
883 "\\ExplSyntaxOff",
884 ]
885 );
886 }
887
888 #[test]
889 fn comment_in_a_consumed_slot_ends_the_fallback_line() {
890 let got = statements(
891 "\\ExplSyntaxOn\n\\exp_after:wN \\foo \\tl_set:Nn \\l_a\n% doc\n{ x } \\group_begin:\n\\ExplSyntaxOff\n",
892 );
893 assert_eq!(
894 got,
895 vec![
896 "\\ExplSyntaxOn",
897 "\\exp_after:wN \\foo \\tl_set:Nn \\l_a % doc { x }",
898 "\\group_begin:",
899 "\\ExplSyntaxOff",
900 ]
901 );
902 }
903
904 #[test]
905 fn unknown_head_falls_back_to_its_line() {
906 let got = statements(
907 "\\ExplSyntaxOn\n\\exp_after:wN \\foo \\tl_set:Nn \\l_a { x }\n\\group_begin:\n\\ExplSyntaxOff\n",
908 );
909 assert_eq!(
910 got,
911 vec![
912 "\\ExplSyntaxOn",
913 "\\exp_after:wN \\foo \\tl_set:Nn \\l_a { x }",
914 "\\group_begin:",
915 "\\ExplSyntaxOff",
916 ]
917 );
918 }
919
920 #[test]
921 fn shape_mismatch_falls_back() {
922 let got = statements("\\ExplSyntaxOn\n\\tl_set:Nn\n\\l_a\n\\ExplSyntaxOff\n");
923 assert_eq!(
924 got,
925 vec!["\\ExplSyntaxOn", "\\tl_set:Nn", "\\l_a", "\\ExplSyntaxOff"]
926 );
927 }
928
929 #[test]
930 fn trailing_comment_rides_the_statement() {
931 let got = statements("\\ExplSyntaxOn\n\\tl_set:Nn \\l_a { x } % note\n\\ExplSyntaxOff\n");
932 assert_eq!(
933 got,
934 vec![
935 "\\ExplSyntaxOn",
936 "\\tl_set:Nn \\l_a { x } % note",
937 "\\ExplSyntaxOff",
938 ]
939 );
940 }
941
942 #[test]
943 fn leftover_attached_group_rides_the_statement() {
944 let got = statements("\\ExplSyntaxOn\n\\use:n { a } { b }\n\\ExplSyntaxOff\n");
945 assert_eq!(
946 got,
947 vec!["\\ExplSyntaxOn", "\\use:n { a } { b }", "\\ExplSyntaxOff"]
948 );
949 }
950
951 #[test]
952 fn conditional_call_is_one_unit() {
953 let got = statements(
954 "\\ExplSyntaxOn\n\\str_if_eq:nnTF { a } { b }\n { yes }\n { no }\n\\ExplSyntaxOff\n",
955 );
956 assert_eq!(
957 got,
958 vec![
959 "\\ExplSyntaxOn",
960 "\\str_if_eq:nnTF { a } { b } { yes } { no }",
961 "\\ExplSyntaxOff",
962 ]
963 );
964 }
965
966 fn branch_texts(src: &str, head: usize) -> Option<Vec<String>> {
967 let parsed = parse(src);
968 assert!(parsed.errors.is_empty(), "test source should parse cleanly");
969 let root = SyntaxNode::new_root(parsed.green);
970 let para = root
971 .children()
972 .find(|n| n.kind() == SyntaxKind::PARAGRAPH)
973 .expect("a paragraph");
974 let elements: Vec<SyntaxElement> = para.children_with_tokens().collect();
975 let unit = expl3_unit(&elements, head)?;
976 Some(
977 unit.branches
978 .iter()
979 .map(|range| normalize(&root.text().slice(*range).to_string()))
980 .collect(),
981 )
982 }
983
984 fn head_of(src: &str, name: &str) -> usize {
985 let parsed = parse(src);
986 let root = SyntaxNode::new_root(parsed.green);
987 let para = root
988 .children()
989 .find(|n| n.kind() == SyntaxKind::PARAGRAPH)
990 .expect("a paragraph");
991 para.children_with_tokens()
992 .position(|el| {
993 el.as_node().is_some_and(|n| {
994 n.kind() == SyntaxKind::COMMAND
995 && command_name(n).is_some_and(|got| got == name)
996 })
997 })
998 .unwrap_or_else(|| panic!("no command named {name}"))
999 }
1000
1001 #[test]
1002 fn branches_are_resolved_wherever_attachment_put_them() {
1003 let head_attached = "\\ExplSyntaxOn\n\\tl_if_empty:nTF {#1} { T } { F }\n";
1004 assert_eq!(
1005 branch_texts(head_attached, head_of(head_attached, "tl_if_empty:nTF")),
1006 Some(vec!["{ T }".to_string(), "{ F }".to_string()])
1007 );
1008
1009 let one_sibling = "\\ExplSyntaxOn\n\\seq_if_in:NnTF \\l_seq {item} { T } { F }\n";
1010 assert_eq!(
1011 branch_texts(one_sibling, head_of(one_sibling, "seq_if_in:NnTF")),
1012 Some(vec!["{ T }".to_string(), "{ F }".to_string()])
1013 );
1014
1015 let two_siblings = "\\ExplSyntaxOn\n\\prop_get:NnNTF \\p {k} \\l { T } { F }\n";
1016 assert_eq!(
1017 branch_texts(two_siblings, head_of(two_siblings, "prop_get:NnNTF")),
1018 Some(vec!["{ T }".to_string(), "{ F }".to_string()])
1019 );
1020
1021 let stream_level = "\\ExplSyntaxOn\n\\int_compare:nNnTF {a} = { 1 } { T } { F }\n";
1022 assert_eq!(
1023 branch_texts(stream_level, head_of(stream_level, "int_compare:nNnTF")),
1024 Some(vec!["{ T }".to_string(), "{ F }".to_string()])
1025 );
1026 }
1027
1028 #[test]
1029 fn a_non_conditional_unit_has_no_branches() {
1030 let src = "\\ExplSyntaxOn\n\\tl_set:Nn \\l_a { x }\n";
1031 assert_eq!(branch_texts(src, head_of(src, "tl_set:Nn")), Some(vec![]));
1032 }
1033
1034 #[test]
1035 fn an_underivable_head_resolves_no_unit() {
1036 let src = "\\ExplSyntaxOn\n\\odd_if:wTF \\a \\b { T } { F }\n";
1037 assert_eq!(branch_texts(src, head_of(src, "odd_if:wTF")), None);
1038 }
1039
1040 #[test]
1041 fn a_blank_line_cut_unit_reports_no_branches() {
1042 let src = "\\ExplSyntaxOn\n\\use:n { \\prop_get:NnNTF \\p {k} \\l { T }\n\n{ F } }\n";
1043 let parsed = parse(src);
1044 assert!(parsed.errors.is_empty());
1045 let root = SyntaxNode::new_root(parsed.green);
1046 let group = root
1047 .descendants()
1048 .find(|n| n.kind() == SyntaxKind::GROUP)
1049 .expect("a group");
1050 let body: Vec<SyntaxElement> = group
1051 .children_with_tokens()
1052 .filter(|el| !matches!(el.kind(), SyntaxKind::L_BRACE | SyntaxKind::R_BRACE))
1053 .collect();
1054 let head = body
1055 .iter()
1056 .position(|el| el.as_node().is_some())
1057 .expect("the head command");
1058 let unit = expl3_unit(&body, head).expect("the partial unit still resolves");
1059 assert_eq!(unit.branches, vec![]);
1060 }
1061
1062 #[test]
1063 fn blank_line_ends_the_unit() {
1064 let src = "\\ExplSyntaxOn\n\\use:n { \\tl_set:Nn \\l_a\n\n { x } }\n\\ExplSyntaxOff\n";
1065 let parsed = parse(src);
1066 assert!(parsed.errors.is_empty());
1067 let root = SyntaxNode::new_root(parsed.green);
1068 let group = root
1069 .descendants()
1070 .find(|n| n.kind() == SyntaxKind::GROUP)
1071 .expect("a group");
1072 let body: Vec<SyntaxElement> = group
1073 .children_with_tokens()
1074 .filter(|el| !matches!(el.kind(), SyntaxKind::L_BRACE | SyntaxKind::R_BRACE))
1075 .collect();
1076 assert_eq!(statement_texts(&body), vec!["\\tl_set:Nn \\l_a", "{ x }"]);
1077 }
1078
1079 #[test]
1080 fn guard_mid_unit_aborts_to_fallback() {
1081 use crate::parser::lexer::LexConfig;
1082 use crate::parser::{LatexFlavor, parse_with_flavor};
1083 let src = "% \\begin{macrocode}\n\\ExplSyntaxOn\n\\tl_set:Nn \\l_a\n%<latexrelease> { x }\n\\ExplSyntaxOff\n% \\end{macrocode}\n";
1084 let config = LexConfig {
1085 flavor: LatexFlavor::Package,
1086 dtx: true,
1087 };
1088 let parsed = parse_with_flavor(src, config);
1089 assert!(parsed.errors.is_empty(), "test source should parse cleanly");
1090 let root = SyntaxNode::new_root(parsed.green);
1091 let para = root
1092 .descendants()
1093 .find(|n| n.kind() == SyntaxKind::PARAGRAPH)
1094 .expect("a paragraph");
1095 let elements: Vec<SyntaxElement> = para.children_with_tokens().collect();
1096 let map = segment_expl_statements(&elements);
1097 assert_eq!(
1098 statement_texts(&elements),
1099 vec![
1100 "\\ExplSyntaxOn",
1101 "\\tl_set:Nn \\l_a %<latexrelease> { x }",
1102 "\\ExplSyntaxOff",
1103 ]
1104 );
1105 let guarded_end = elements
1106 .iter()
1107 .position(|el| el.to_string().contains("latexrelease"))
1108 .expect("the guarded sibling");
1109 assert!(
1110 map.is_fallback(guarded_end),
1111 "the aborted unit must be a fallback statement"
1112 );
1113 }
1114
1115 #[test]
1116 fn e_and_f_letters_consume_braced_groups() {
1117 let got = statements(
1118 "\\ExplSyntaxOn\n\\tl_set:Ne \\l_a\n { x }\n\\tl_set:Nf \\l_b\n { y }\n\\ExplSyntaxOff\n",
1119 );
1120 assert_eq!(
1121 got,
1122 vec![
1123 "\\ExplSyntaxOn",
1124 "\\tl_set:Ne \\l_a { x }",
1125 "\\tl_set:Nf \\l_b { y }",
1126 "\\ExplSyntaxOff",
1127 ]
1128 );
1129 }
1130
1131 #[test]
1132 fn stream_ending_mid_unit_falls_back() {
1133 let src = "\\ExplSyntaxOn\n\\use:n { \\tl_set:Nn \\l_a }\n\\ExplSyntaxOff\n";
1134 let parsed = parse(src);
1135 assert!(parsed.errors.is_empty());
1136 let root = SyntaxNode::new_root(parsed.green);
1137 let group = root
1138 .descendants()
1139 .find(|n| n.kind() == SyntaxKind::GROUP)
1140 .expect("a group");
1141 let body: Vec<SyntaxElement> = group
1142 .children_with_tokens()
1143 .filter(|el| !matches!(el.kind(), SyntaxKind::L_BRACE | SyntaxKind::R_BRACE))
1144 .collect();
1145 let map = segment_expl_statements(&body);
1146 assert_eq!(statement_texts(&body), vec!["\\tl_set:Nn \\l_a"]);
1147 let head = body
1148 .iter()
1149 .position(|el| el.as_node().is_some())
1150 .expect("the head command");
1151 assert!(
1152 map.is_fallback(head),
1153 "a unit cut off by the stream end must be a fallback statement"
1154 );
1155 }
1156
1157 #[test]
1158 fn a_multi_line_group_node_does_not_end_a_fallback_line() {
1159 let src = "\\ExplSyntaxOn\n\
1160 \\int_do_until:w { \\l_tmpa_int } > {#2}\n\
1161 { \\lipsum_add:V { \\l_tmpa_int }\n\
1162 \\int_incr:N \\l_tmpa_int } \\tl_put_right:NV \\l_a \\l_b\n\
1163 \\ExplSyntaxOff\n";
1164 let parsed = parse(src);
1165 assert!(parsed.errors.is_empty());
1166 let root = SyntaxNode::new_root(parsed.green);
1167 let elements: Vec<SyntaxElement> = root
1168 .first_child()
1169 .expect("the paragraph")
1170 .children_with_tokens()
1171 .collect();
1172 let map = segment_expl_statements(&elements);
1173
1174 assert_eq!(
1175 statement_texts(&elements),
1176 vec![
1177 "\\ExplSyntaxOn",
1178 "\\int_do_until:w { \\l_tmpa_int } > {#2}",
1179 "{ \\lipsum_add:V { \\l_tmpa_int } \\int_incr:N \\l_tmpa_int } \
1180 \\tl_put_right:NV \\l_a \\l_b",
1181 "\\ExplSyntaxOff",
1182 ]
1183 );
1184
1185 let group = elements
1186 .iter()
1187 .position(|el| el.kind() == SyntaxKind::GROUP && el.to_string().contains('\n'))
1188 .expect("the multi-line group");
1189 assert!(
1190 map.is_fallback(group),
1191 "the group belongs to a fallback statement"
1192 );
1193 assert!(
1194 !map.boundary_after(group),
1195 "a multi-line group's own newlines must not end the fallback line"
1196 );
1197
1198 let head = elements
1199 .iter()
1200 .skip(group)
1201 .position(|el| {
1202 el.as_node()
1203 .is_some_and(|n| n.kind() == SyntaxKind::COMMAND)
1204 })
1205 .map(|off| group + off)
1206 .expect("the trailing recognized head");
1207 assert!(
1208 map.glue_before(head),
1209 "a recognized head mid-fallback-line owes an unbreakable gap"
1210 );
1211 }
1212
1213 #[test]
1214 fn own_line_comment_in_attached_span_rides_the_sibling() {
1215 let got =
1216 statements("\\ExplSyntaxOn\n\\tl_set:Nn \\l_a\n% note\n { x }\n\\ExplSyntaxOff\n");
1217 assert_eq!(
1218 got,
1219 vec![
1220 "\\ExplSyntaxOn",
1221 "\\tl_set:Nn \\l_a % note { x }",
1222 "\\ExplSyntaxOff",
1223 ]
1224 );
1225 }
1226
1227 #[test]
1228 fn own_line_comment_at_sibling_level_ends_the_unit() {
1229 let got = statements(
1230 "\\ExplSyntaxOn\n\\cs_new:Npn \\foo:n\n% note\n#1 { body }\n\\ExplSyntaxOff\n",
1231 );
1232 assert_eq!(
1233 got,
1234 vec![
1235 "\\ExplSyntaxOn",
1236 "\\cs_new:Npn \\foo:n",
1237 "% note",
1238 "#1 { body }",
1239 "\\ExplSyntaxOff",
1240 ]
1241 );
1242 }
1243}