1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
//! Trivia scanning and the leading-comment bind.
//!
//! Every blank-line and comment-attachment question the grammar asks is decided
//! from one walk of a trivia run ([`Parser::scan_trivia`]) rather than five
//! near-identical ones. Comments bind forward into the construct they annotate,
//! whitespace floats, and a blank line breaks the bind.
use super::{BEGIN_CMD, END_CMD, Parser};
use crate::syntax::SyntaxKind;
/// How the shared trivia scanner ([`Parser::scan_trivia`]) treats a `%` comment.
#[derive(Clone, Copy, PartialEq, Eq)]
pub(super) enum CommentMode {
/// A comment is content that occupies its own line: it resets the newline
/// run (without undoing a blank line already seen) and the scan continues
/// past it. Used everywhere paragraph structure and leading-comment binds are
/// decided.
Skip,
/// A comment stops the scan and is reported as the next meaningful token.
/// Used by [`Parser::at_script`], where a comment ends the line so a `^`/`_`
/// script never binds across it.
Stop,
}
/// Newlines in a trivia run that make it a **blank line** — TeX's `\par`
/// boundary. One newline only ends the current line.
pub(super) const BLANK_LINE_NEWLINES: usize = 2;
/// The result of scanning a contiguous trivia run.
pub(super) struct TriviaScan {
/// Index of the next meaningful (non-skipped) token, or `tokens.len()` at EOF.
pub(super) next: usize,
/// Kind of the token at [`Self::next`], or `None` at EOF.
pub(super) next_kind: Option<SyntaxKind>,
/// The run before `next` contains a blank line (≥2 `NEWLINE`s: the `\par`
/// boundary).
pub(super) saw_blank_line: bool,
/// As [`Self::saw_blank_line`], but a `.dtx` docstrip guard line counts as
/// content rather than blank space. Docstrip *deletes* a guard-only line
/// when it strips the file, so `%<*dtx>` between two lines does not part
/// them — read by the shape gates that only ask whether their construct's
/// source ran out mid-shape (issue #71).
pub(super) saw_blank_line_outside_guards: bool,
/// Start index of the leading own-line `%` comment run immediately preceding
/// `next` — the maximal blank-line-free suffix, the start of a
/// leading-comment bind. `None` if that suffix has no own-line comment.
/// Only populated in [`CommentMode::Skip`].
pub(super) comment_start: Option<usize>,
}
impl Parser<'_> {
pub(super) fn is_trivia(k: SyntaxKind) -> bool {
matches!(
k,
SyntaxKind::WHITESPACE
| SyntaxKind::NEWLINE
| SyntaxKind::COMMENT
| SyntaxKind::DOC_MARGIN
| SyntaxKind::GUARD
)
}
pub(super) fn skip_trivia(&mut self) {
while self.kind().is_some_and(Self::is_trivia) {
self.bump();
}
}
/// Scan the contiguous trivia run starting at `from`, classifying each token
/// so the blank-line and leading-comment-bind rules (AGENTS.md #9) can be
/// decided from one walk instead of five near-identical ones. `WHITESPACE`,
/// `DOC_MARGIN`, and `GUARD` float (a `.dtx` margin neither counts as a
/// newline nor resets the run, so a margin-only line `%\n%\n` still reads as a
/// blank line via its two `NEWLINE`s); `NEWLINE`s accumulate into the blank-line
/// (`≥2`) test; a `COMMENT` is handled per [`CommentMode`]. A `GUARD` floats
/// for `saw_blank_line` but breaks the run for
/// [`TriviaScan::saw_blank_line_outside_guards`]. Does not consume.
pub(super) fn scan_trivia(&self, from: usize, comment_mode: CommentMode) -> TriviaScan {
let mut i = from;
let mut newlines = 0;
let mut guard_newlines = 0;
let mut saw_blank_line = false;
let mut saw_blank_line_outside_guards = false;
let mut comment_start = None;
while let Some(t) = self.tokens.get(i) {
match t.kind {
SyntaxKind::NEWLINE => {
newlines += 1;
guard_newlines += 1;
if newlines >= BLANK_LINE_NEWLINES {
saw_blank_line = true;
// A blank line breaks a leading-comment bind: only a
// comment *after* it can still bind, so drop any comment
// seen before it.
comment_start = None;
}
if guard_newlines >= BLANK_LINE_NEWLINES {
saw_blank_line_outside_guards = true;
}
}
SyntaxKind::WHITESPACE | SyntaxKind::DOC_MARGIN => {}
// A docstrip guard floats like a margin for the layout rules
// (`saw_blank_line`), but it is *content* on its line — and a
// line docstrip deletes outright when it strips the file, so a
// guard-only line is not a blank line separating what surrounds
// it. Constructs that only need to know whether their source
// ran out mid-shape read `saw_blank_line_outside_guards`
// instead (issue #71).
SyntaxKind::GUARD => guard_newlines = 0,
SyntaxKind::COMMENT if comment_mode == CommentMode::Stop => break,
// A comment occupies its own line: it is content, not blank space,
// so it resets the newline run (without undoing a blank line
// already seen) and, if it starts its line, opens a
// leading-comment bind.
SyntaxKind::COMMENT => {
newlines = 0;
guard_newlines = 0;
if comment_start.is_none() && self.comment_starts_line(i) {
comment_start = Some(i);
}
}
_ => break,
}
i += 1;
}
TriviaScan {
next: i,
next_kind: self.tokens.get(i).map(|t| t.kind),
saw_blank_line,
saw_blank_line_outside_guards,
comment_start,
}
}
/// Peek the kind of the next non-trivia token and whether the intervening
/// trivia contains a paragraph break (a blank line, i.e. ≥2 newlines).
/// Does not consume.
pub(super) fn peek_meaningful(&self) -> (Option<SyntaxKind>, bool) {
let s = self.scan_trivia(self.pos, CommentMode::Skip);
(s.next_kind, s.saw_blank_line)
}
/// Text of the next non-trivia token at/after `self.pos`, if any. Does not
/// consume. Used to distinguish a verbatim-argument `VERB` from a standalone
/// `\verb…` token (see `attach_arguments`).
pub(super) fn peek_meaningful_text(&self) -> Option<&str> {
let mut i = self.pos;
while let Some(t) = self.tokens.get(i) {
if !Self::is_trivia(t.kind) {
return Some(t.text.as_str());
}
i += 1;
}
None
}
/// True if a paragraph break (blank line) begins at the current position.
pub(super) fn at_paragraph_break(&self) -> bool {
self.scan_trivia(self.pos, CommentMode::Skip).saw_blank_line
}
/// [`Self::at_paragraph_break`], but blind to `.dtx` docstrip guard lines:
/// a `%<*dtx>`/`%</dtx>` pair on its own lines is not the blank line it
/// looks like, because docstrip deletes those lines outright. Used by the
/// bail-out anchors of constructs that legitimately span a guarded block —
/// `\ProvidesPackage{…}` and its `[…date…]` optional, split across
/// `%<package>`/`%<*dtx>` variants (rotating.dtx, issue #71).
pub(super) fn at_paragraph_break_outside_guards(&self) -> bool {
self.scan_trivia(self.pos, CommentMode::Skip)
.saw_blank_line_outside_guards
}
/// True if the comment at `pos` starts its own line: scanning back over
/// inline whitespace only, the preceding token is a `NEWLINE` or the start of
/// input. A same-line trailing comment (`\foo % x`) returns `false` and never
/// binds forward (see [`Self::binding_run`]).
pub(super) fn comment_starts_line(&self, pos: usize) -> bool {
let mut i = pos;
while i > 0 {
i -= 1;
match self.tokens[i].kind {
// A `.dtx` margin or guard is skipped like whitespace when deciding
// whether a comment owns its line (neither is itself the comment).
SyntaxKind::WHITESPACE | SyntaxKind::DOC_MARGIN | SyntaxKind::GUARD => {
continue;
}
SyntaxKind::NEWLINE => return true,
_ => return false,
}
}
true
}
/// If the trivia run at `from` ends in a `%` comment run that binds *leading*
/// into a following documentable construct, return
/// `(comment_start, construct_pos, construct_kind)`:
/// - `comment_start` — index of the first own-line comment of the binding run
/// (the maximal blank-line-free suffix; trivia before it floats),
/// - `construct_pos` — index of the construct's control word,
/// - `construct_kind` — `ENVIRONMENT` for `\begin`, otherwise `COMMAND`.
///
/// Returns `None` when the run has no own-line comment, a blank line separates
/// the comment from the construct, or the next non-trivia token is not a
/// documentable construct. Mirrors rust-analyzer's `n_attached_trivias`
/// (AGENTS.md #9): comments bind forward to the item they annotate, a blank
/// line breaks the bind, and a same-line trailing comment never binds.
///
/// One deliberate divergence: RA peeks *past* a blank line and keeps attaching
/// when the next comment is an outer doc comment (`///`/`//!`). LaTeX's single
/// `%` carries no such intent marker, so we always stop at the blank line (only
/// the maximal blank-line-free suffix binds). See AGENTS.md #9;
/// `comment_after_blank_line_still_binds` (`tests/parser.rs`) pins the divergence.
pub(super) fn binding_run(&self, from: usize) -> Option<(usize, usize, SyntaxKind)> {
let s = self.scan_trivia(from, CommentMode::Skip);
let start = s.comment_start?;
if s.next_kind != Some(SyntaxKind::CONTROL_WORD) {
return None;
}
// An alias closer terminates the body just as `\end` does, so it is not a
// construct a preceding comment run may bind to. Without this the run would
// classify as `COMMAND`, the body loop would open a `DOC_COMMENT` and then
// call `element` *at the closer* — consuming it inside the body, so the
// environment never closes.
if self.alias_end.is_some_and(|end| s.next >= end) {
return None;
}
let kind = match self.tokens[s.next].text.as_str() {
BEGIN_CMD => SyntaxKind::ENVIRONMENT,
END_CMD => return None,
_ => SyntaxKind::COMMAND,
};
Some((start, s.next, kind))
}
}