brink_syntax/segment.rs
1//! Lexer-driven file segmentation for per-knot incremental lowering
2//! (issue #3084, `docs/per-knot-incremental-lowering-spec.md` §3 step 1).
3//!
4//! [`segment_file`] splits an ink source file into a leading header
5//! segment plus one segment per top-level knot (and per top-level stitch
6//! before the first knot — the ones lowering promotes to knots). Segments
7//! **tile** the file: every byte belongs to exactly one segment, in
8//! source order, so downstream assembly can rebase per-segment output by
9//! each segment's current offset.
10//!
11//! The boundary decision mirrors the parser's dispatch rule exactly — it
12//! must, because per-segment parse output has to match the corresponding
13//! subtree of the whole-file parse byte-for-byte:
14//!
15//! - A header starts at a **dispatch point**: the file start, or after a
16//! `NEWLINE` at interpolation-brace depth zero, with trivia
17//! (whitespace, `//`, `/* … */`) skipped. This is `source_file`'s /
18//! `knot_body`'s loop shape: a `==` inside a prose line, a string, or a
19//! `/* … */` block comment (including a multi-line or unterminated one
20//! — the lexer scans those to `*/` or EOF as one token) never splits.
21//! - Brace depth tracks `{`/`}` tokens because a multiline block's inner
22//! lines are consumed inside one statement — the parser reaches no
23//! dispatch point there, and its inner loops do not break on knot
24//! headers, so an unterminated `{` swallowing the rest of the file is
25//! mirrored here by depth never returning to zero.
26//! - `EQ_EQ` at a dispatch point opens a knot (`at_knot`); a single `EQ`
27//! whose next non-trivia token is neither `EQ` nor `GT` opens a
28//! top-level stitch (`at_stitch`) — only until the first knot, after
29//! which stitch headers are internal to their knot's segment.
30//! - A segment's range extends **backward over the contiguous `///`
31//! doc-comment block** preceding its header, mirroring
32//! `collect_doc_lines` in `brink-ir`'s lowering (walk back over
33//! whitespace and newlines, attach `///` line comments, break on a
34//! blank-line gap of two newlines, a plain `//` comment, or any other
35//! token): a doc block is structurally part of the declaration it
36//! precedes, so it must travel with the knot's segment.
37
38use rowan::{TextRange, TextSize};
39
40use crate::SyntaxKind;
41use crate::lexer::lex;
42
43/// What a [`Segment`] covers.
44#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
45pub enum SegmentKind {
46 /// Everything before the first knot/stitch header: declarations,
47 /// includes, root weave content. Always present (possibly empty).
48 Header,
49 /// One top-level knot (`== name …`), doc block included.
50 Knot,
51 /// One top-level stitch (`= name`, before any knot), doc block
52 /// included — lowering promotes these to knots.
53 TopLevelStitch,
54}
55
56/// One contiguous slice of the file. Produced by [`segment_file`];
57/// segments tile the file in source order.
58#[derive(Debug, Clone, PartialEq, Eq, Hash)]
59pub struct Segment {
60 pub kind: SegmentKind,
61 /// The byte range this segment covers, doc-block extension included.
62 /// Ranges TILE the file (offset bookkeeping); the text a consumer
63 /// should PARSE is [`lowered_range`](Self::lowered_range).
64 pub range: TextRange,
65 /// The range to parse for this segment: `range` extended through the
66 /// trailing trivia up to the NEXT segment's header token (or EOF).
67 /// The whole-file parse absorbs the trivia before a knot header —
68 /// blank lines, comments, and the next knot's `///` doc block — into
69 /// the PRECEDING knot's node, so a fragment must end at the next
70 /// header for its node ranges to match the whole-file tree
71 /// byte-for-byte. Where the next boundary carries a doc block, this
72 /// makes adjacent `lowered_range`s OVERLAP on the doc bytes: the doc
73 /// block is trailing trivia to this segment and doc attachment to the
74 /// next — both readings are what the whole-file parse does.
75 pub lowered_range: TextRange,
76 /// The byte offset of the header's first `=` token — the position the
77 /// whole-file parse gives the corresponding `KNOT_HEADER`/
78 /// `STITCH_HEADER` node. `None` for the header segment.
79 pub header_start: Option<TextSize>,
80}
81
82/// Split `source` into a header segment plus one segment per top-level
83/// knot / top-level stitch. See the module doc for the boundary rules.
84#[must_use]
85pub fn segment_file(source: &str) -> Vec<Segment> {
86 let tokens = lex(source);
87
88 // Token start offsets (the lexer is lossless: slices tile the file).
89 let mut starts: Vec<TextSize> = Vec::with_capacity(tokens.len());
90 let mut pos = TextSize::from(0);
91 for (_, text) in &tokens {
92 starts.push(pos);
93 pos += TextSize::of(*text);
94 }
95 let total = pos;
96
97 // (cut, header_start, kind) per boundary, in source order.
98 let mut boundaries: Vec<(TextSize, TextSize, SegmentKind)> = Vec::new();
99 let mut brace_depth: u32 = 0;
100 let mut at_dispatch = true;
101 let mut seen_knot = false;
102
103 let mut i = 0;
104 while i < tokens.len() {
105 let kind = tokens[i].0;
106 if at_dispatch {
107 match kind {
108 _ if kind.is_trivia() => {
109 i += 1;
110 continue;
111 }
112 // An empty line: consume and stay at the dispatch point.
113 SyntaxKind::NEWLINE => {
114 i += 1;
115 continue;
116 }
117 SyntaxKind::EQ_EQ => {
118 boundaries.push((
119 doc_extended_start(&tokens, &starts, i),
120 starts[i],
121 SegmentKind::Knot,
122 ));
123 seen_knot = true;
124 at_dispatch = false;
125 i += 1;
126 continue;
127 }
128 SyntaxKind::EQ if !seen_knot && at_stitch_lookahead(&tokens, i) => {
129 boundaries.push((
130 doc_extended_start(&tokens, &starts, i),
131 starts[i],
132 SegmentKind::TopLevelStitch,
133 ));
134 at_dispatch = false;
135 i += 1;
136 continue;
137 }
138 // Any other token starts an ordinary statement/line —
139 // fall through to the normal (non-dispatch) handling of
140 // this same token.
141 _ => at_dispatch = false,
142 }
143 }
144 match kind {
145 SyntaxKind::L_BRACE => brace_depth += 1,
146 SyntaxKind::R_BRACE => brace_depth = brace_depth.saturating_sub(1),
147 SyntaxKind::NEWLINE if brace_depth == 0 => at_dispatch = true,
148 _ => {}
149 }
150 i += 1;
151 }
152
153 // Assemble tiling segments.
154 let first_cut = boundaries.first().map_or(total, |b| b.0);
155 let first_header = boundaries.first().map_or(total, |b| b.1);
156 let mut segments = Vec::with_capacity(boundaries.len() + 1);
157 segments.push(Segment {
158 kind: SegmentKind::Header,
159 range: TextRange::new(TextSize::from(0), first_cut),
160 lowered_range: TextRange::new(TextSize::from(0), first_header),
161 header_start: None,
162 });
163 for (idx, &(cut, header_start, kind)) in boundaries.iter().enumerate() {
164 let end = boundaries.get(idx + 1).map_or(total, |b| b.0);
165 let lowered_end = boundaries.get(idx + 1).map_or(total, |b| b.1);
166 segments.push(Segment {
167 kind,
168 range: TextRange::new(cut, end),
169 lowered_range: TextRange::new(cut, lowered_end),
170 header_start: Some(header_start),
171 });
172 }
173 segments
174}
175
176/// The `at_stitch` mirror: the next non-trivia token after the `EQ` is
177/// neither `EQ` (a `= =` run is not a stitch) nor `GT` (`=>`).
178fn at_stitch_lookahead(tokens: &[(SyntaxKind, &str)], eq_idx: usize) -> bool {
179 let mut j = eq_idx + 1;
180 while j < tokens.len() && tokens[j].0.is_trivia() {
181 j += 1;
182 }
183 !matches!(
184 tokens.get(j).map(|t| t.0),
185 Some(SyntaxKind::EQ | SyntaxKind::GT)
186 )
187}
188
189/// Extend a header boundary backward over the contiguous `///` doc block
190/// preceding it — the exact walk `collect_doc_lines` performs during
191/// lowering (skip whitespace; count newlines, two in a row is a blank
192/// line and ends the block; a `///` line comment attaches and resets the
193/// newline count; a plain `//` comment or any other token ends the walk).
194/// Returns the cut position: the start of the earliest attached `///`
195/// token, or the header token's own start when no doc block precedes it.
196fn doc_extended_start(
197 tokens: &[(SyntaxKind, &str)],
198 starts: &[TextSize],
199 header_idx: usize,
200) -> TextSize {
201 let mut cut = starts[header_idx];
202 let mut newlines = 0u32;
203 let mut j = header_idx;
204 while j > 0 {
205 j -= 1;
206 match tokens[j].0 {
207 SyntaxKind::WHITESPACE => {}
208 SyntaxKind::NEWLINE => {
209 newlines += 1;
210 if newlines >= 2 {
211 break;
212 }
213 }
214 SyntaxKind::LINE_COMMENT if tokens[j].1.starts_with("///") => {
215 newlines = 0;
216 cut = starts[j];
217 }
218 _ => break,
219 }
220 }
221 cut
222}
223
224#[cfg(test)]
225mod tests;