varar_core/
scanner.rs

1//! Turns raw Markdown into a flat list of [`Block`] nodes — port of `scanner.ts`
2//! / `Scanner.java`. Offsets in stored spans are UTF-16 code units; the line
3//! splitter keeps a running (byte, UTF-16) dual cursor and per-line regex offsets
4//! are converted from bytes to UTF-16.
5//!
6//! Following `Scanner.java`, [`scan`] takes no plugins parameter — there is no
7//! scanner-plugin hook in this port.
8
9use crate::ast::{
10    Block, Blockquote, Fence, Heading, ListItem, Paragraph, Row, SegmentOffset, Table,
11    ThematicBreak,
12};
13use crate::offsets::{java_trim, utf16_index, utf16_len};
14use crate::span::Span;
15use crate::table_cells::parse_row_cells;
16use regex::Regex;
17use std::sync::LazyLock;
18
19/// One line of source, with its UTF-16 and byte offsets in the full source.
20struct RawLine {
21    text: String,
22    start_offset: usize,
23    end_offset: usize,
24    start_byte: usize,
25    end_byte: usize,
26}
27
28// `\1` backreference is expanded into three alternatives (the `regex` crate has
29// no backreferences); otherwise these mirror the Java patterns. `[0-9]` keeps the
30// ordered-list digit class ASCII.
31static THEMATIC_RE: LazyLock<Regex> = LazyLock::new(|| {
32    Regex::new(r"^\s*(?:-(?:\s*-){2,}|\*(?:\s*\*){2,}|_(?:\s*_){2,})\s*$").unwrap()
33});
34static UL_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"^(\s*)([-*+])\s+(.*)$").unwrap());
35static OL_RE: LazyLock<Regex> =
36    LazyLock::new(|| Regex::new(r"^(\s*)([0-9]+)([.)])\s+(.*)$").unwrap());
37static BQ_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"^>\s?(.*)$").unwrap());
38static FENCE_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"^(`{3,})\s*(\S*)\s*$").unwrap());
39static ROW_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"^\|(.+)\|\s*$").unwrap());
40static DELIM_RE: LazyLock<Regex> =
41    LazyLock::new(|| Regex::new(r"^\|\s*:?-+:?\s*(\|\s*:?-+:?\s*)*\|\s*$").unwrap());
42static HEADING_RE: LazyLock<Regex> =
43    LazyLock::new(|| Regex::new(r"^(#{1,6})\s+(.*?)(?:\s+#+)?\s*$").unwrap());
44static HEADING_PREFIX_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"^#{1,6}\s+").unwrap());
45
46/// Scans `source` into a list of [`Block`] nodes.
47pub fn scan(source: &str) -> Vec<Block> {
48    let lines = split_lines(source);
49    let mut blocks = Vec::new();
50
51    let mut i = 0;
52    while i < lines.len() {
53        if java_trim(&lines[i].text).is_empty() {
54            i += 1;
55            continue;
56        }
57        if let Some((fence, next)) = try_fence(source, &lines, i) {
58            blocks.push(Block::Fence(fence));
59            i = next;
60            continue;
61        }
62        if let Some((table, next)) = try_table(source, &lines, i) {
63            blocks.push(Block::Table(table));
64            i = next;
65            continue;
66        }
67        if let Some(tb) = try_thematic_break(source, &lines[i]) {
68            blocks.push(Block::ThematicBreak(tb));
69            i += 1;
70            continue;
71        }
72        if let Some((quote, next)) = try_blockquote(source, &lines, i) {
73            blocks.push(Block::Blockquote(quote));
74            i = next;
75            continue;
76        }
77        if let Some(heading) = try_heading(source, &lines[i]) {
78            blocks.push(Block::Heading(heading));
79            i += 1;
80            continue;
81        }
82        if let Some(item) = try_list_item(source, &lines[i]) {
83            blocks.push(Block::ListItem(item));
84            i += 1;
85            continue;
86        }
87        let (paragraph, next) = consume_paragraph(source, &lines, i);
88        blocks.push(Block::Paragraph(paragraph));
89        i = next;
90    }
91    blocks
92}
93
94fn split_lines(source: &str) -> Vec<RawLine> {
95    let mut out = Vec::new();
96    let mut byte_start = 0;
97    let mut u16_start = 0;
98    let mut u16 = 0;
99    for (byte_i, c) in source.char_indices() {
100        if c == '\n' {
101            out.push(RawLine {
102                text: source[byte_start..byte_i].to_string(),
103                start_offset: u16_start,
104                end_offset: u16,
105                start_byte: byte_start,
106                end_byte: byte_i,
107            });
108            byte_start = byte_i + 1;
109            u16_start = u16 + 1;
110        }
111        u16 += c.len_utf16();
112    }
113    out.push(RawLine {
114        text: source[byte_start..].to_string(),
115        start_offset: u16_start,
116        end_offset: u16,
117        start_byte: byte_start,
118        end_byte: source.len(),
119    });
120    out
121}
122
123fn try_thematic_break(source: &str, line: &RawLine) -> Option<ThematicBreak> {
124    if !THEMATIC_RE.is_match(&line.text) {
125        return None;
126    }
127    Some(ThematicBreak {
128        span: Span::from_offsets(source, line.start_offset, line.end_offset),
129    })
130}
131
132fn try_heading(source: &str, line: &RawLine) -> Option<Heading> {
133    let m = HEADING_RE.captures(&line.text)?;
134    let hashes = m.get(1).unwrap().as_str();
135    let text = java_trim(m.get(2).unwrap().as_str()).to_string();
136    Some(Heading {
137        level: hashes.len(),
138        text,
139        span: Span::from_offsets(source, line.start_offset, line.end_offset),
140    })
141}
142
143fn try_list_item(source: &str, line: &RawLine) -> Option<ListItem> {
144    if let Some(ul) = UL_RE.captures(&line.text) {
145        let text = ul.get(3).unwrap().as_str();
146        let marker_start = line.start_offset + utf16_len(ul.get(1).unwrap().as_str());
147        let marker_end = marker_start + utf16_len(ul.get(2).unwrap().as_str());
148        let text_start = line.start_offset + utf16_index(&line.text, line.text.find(text).unwrap());
149        return Some(ListItem {
150            text: text.to_string(),
151            span: Span::from_offsets(source, line.start_offset, line.end_offset),
152            segment_map: vec![SegmentOffset::new(0, text_start)],
153            ordered: false,
154            marker_span: Span::from_offsets(source, marker_start, marker_end),
155        });
156    }
157    if let Some(ol) = OL_RE.captures(&line.text) {
158        let text = ol.get(4).unwrap().as_str();
159        let marker_start = line.start_offset + utf16_len(ol.get(1).unwrap().as_str());
160        let marker_end = marker_start
161            + utf16_len(ol.get(2).unwrap().as_str())
162            + utf16_len(ol.get(3).unwrap().as_str());
163        let text_start = line.start_offset + utf16_index(&line.text, line.text.find(text).unwrap());
164        return Some(ListItem {
165            text: text.to_string(),
166            span: Span::from_offsets(source, line.start_offset, line.end_offset),
167            segment_map: vec![SegmentOffset::new(0, text_start)],
168            ordered: true,
169            marker_span: Span::from_offsets(source, marker_start, marker_end),
170        });
171    }
172    None
173}
174
175fn try_blockquote(
176    source: &str,
177    lines: &[RawLine],
178    start_idx: usize,
179) -> Option<(Blockquote, usize)> {
180    let first = &lines[start_idx];
181    let m = BQ_RE.captures(&first.text)?;
182    let first_segment = m.get(1).unwrap().as_str().to_string();
183
184    let mut segments = vec![first_segment.clone()];
185    let mut segment_map = vec![SegmentOffset::new(
186        0,
187        first.start_offset + utf16_index(&first.text, first.text.find(&first_segment).unwrap()),
188    )];
189    let mut joined_text_offset = utf16_len(&first_segment);
190
191    let mut i = start_idx + 1;
192    let mut end_offset = first.end_offset;
193    while i < lines.len() {
194        let ln = &lines[i];
195        let Some(next) = BQ_RE.captures(&ln.text) else {
196            break;
197        };
198        let segment = next.get(1).unwrap().as_str().to_string();
199        joined_text_offset += 1; // newline separator
200        segment_map.push(SegmentOffset::new(
201            joined_text_offset,
202            ln.start_offset + utf16_index(&ln.text, ln.text.find(&segment).unwrap()),
203        ));
204        joined_text_offset += utf16_len(&segment);
205        segments.push(segment);
206        end_offset = ln.end_offset;
207        i += 1;
208    }
209    let quote = Blockquote {
210        text: segments.join("\n"),
211        span: Span::from_offsets(source, first.start_offset, end_offset),
212        segment_map,
213    };
214    Some((quote, i))
215}
216
217fn consume_paragraph(source: &str, lines: &[RawLine], start_idx: usize) -> (Paragraph, usize) {
218    let first = &lines[start_idx];
219    let mut end_idx = start_idx;
220    while end_idx + 1 < lines.len() {
221        let candidate = &lines[end_idx + 1];
222        let t = &candidate.text;
223        if java_trim(t).is_empty()
224            || HEADING_PREFIX_RE.is_match(t)
225            || UL_RE.is_match(t)
226            || OL_RE.is_match(t)
227            || BQ_RE.is_match(t)
228            || FENCE_RE.is_match(t)
229            || ROW_RE.is_match(t)
230            || THEMATIC_RE.is_match(t)
231        {
232            break;
233        }
234        end_idx += 1;
235    }
236    let last = &lines[end_idx];
237    let paragraph = Paragraph {
238        text: source[first.start_byte..last.end_byte].to_string(),
239        span: Span::from_offsets(source, first.start_offset, last.end_offset),
240        segment_map: vec![SegmentOffset::new(0, first.start_offset)],
241    };
242    (paragraph, end_idx + 1)
243}
244
245fn try_fence(source: &str, lines: &[RawLine], start_idx: usize) -> Option<(Fence, usize)> {
246    let start = &lines[start_idx];
247    let open = FENCE_RE.captures(&start.text)?;
248    let fence_marker = open.get(1).unwrap().as_str().to_string();
249    let info = java_trim(open.get(2).unwrap().as_str()).to_string();
250
251    let mut i = start_idx + 1;
252    let mut body_start: Option<(usize, usize)> = None; // (u16, byte)
253    let mut body_end: Option<(usize, usize)> = None;
254    let mut end_offset = start.end_offset;
255    while i < lines.len() {
256        let ln = &lines[i];
257        if let Some(close) = FENCE_RE.captures(&ln.text) {
258            if close.get(1).unwrap().as_str().len() >= fence_marker.len() {
259                end_offset = ln.end_offset;
260                break;
261            }
262        }
263        if body_start.is_none() {
264            body_start = Some((ln.start_offset, ln.start_byte));
265        }
266        // Include the newline that separates this line from the next.
267        body_end = Some((ln.end_offset + 1, ln.end_byte + 1));
268        i += 1;
269    }
270
271    let source_u16 = utf16_len(source);
272    let clamped_end_u16 = body_end.map_or(0, |(u16, _)| u16.min(source_u16));
273    let clamped_end_byte = body_end.map_or(0, |(_, byte)| byte.min(source.len()));
274    let body = match (body_start, body_end) {
275        (Some((_, sb)), Some(_)) => source[sb..clamped_end_byte].to_string(),
276        _ => String::new(),
277    };
278    let fallback = start.end_offset;
279    let body_span = Span::from_offsets(
280        source,
281        body_start.map_or(fallback, |(u16, _)| u16),
282        if body_end.is_some() {
283            clamped_end_u16
284        } else {
285            fallback
286        },
287    );
288    let fence = Fence {
289        span: Span::from_offsets(source, start.start_offset, end_offset),
290        info,
291        body,
292        body_span,
293    };
294    Some((fence, i + 1))
295}
296
297fn try_table(source: &str, lines: &[RawLine], start_idx: usize) -> Option<(Table, usize)> {
298    if start_idx + 1 >= lines.len() {
299        return None;
300    }
301    let header_line = &lines[start_idx];
302    let delim_line = &lines[start_idx + 1];
303    if !ROW_RE.is_match(&header_line.text) || !DELIM_RE.is_match(&delim_line.text) {
304        return None;
305    }
306
307    let header_parsed = parse_row_cells(&header_line.text, header_line.start_offset, source);
308    let header = Row {
309        cells: header_parsed.cells,
310        cell_spans: header_parsed.cell_spans,
311        span: Span::from_offsets(source, header_line.start_offset, header_line.end_offset),
312    };
313
314    let mut rows = Vec::new();
315    let mut i = start_idx + 2;
316    while i < lines.len() {
317        let ln = &lines[i];
318        if !ROW_RE.is_match(&ln.text) {
319            break;
320        }
321        let parsed = parse_row_cells(&ln.text, ln.start_offset, source);
322        rows.push(Row {
323            cells: parsed.cells,
324            cell_spans: parsed.cell_spans,
325            span: Span::from_offsets(source, ln.start_offset, ln.end_offset),
326        });
327        i += 1;
328    }
329    let end_offset = rows
330        .last()
331        .map_or(delim_line.end_offset, |r| r.span.end_offset);
332    let table = Table {
333        span: Span::from_offsets(source, header_line.start_offset, end_offset),
334        header,
335        rows,
336    };
337    Some((table, i))
338}
varar_core/scanner.rs

varar_core/
scanner.rs
