arity_parser/rd/
parser.rs

1//! A mode-aware source scanner and recoverable recursive-descent Rd parser.
2//!
3//! Every token is an untouched slice at the current byte cursor. Recovery never
4//! synthesizes source text, and the nesting guard retains the remaining bytes.
5
6use rowan::{GreenNodeBuilder, ast::AstNode};
7
8use super::ast::Document;
9use super::macros::{self, Items, Mode};
10use super::syntax::{SyntaxKind as K, SyntaxNode};
11use crate::parser::ParseDiagnostic;
12
13/// A complete source tree and recoverable, byte-ranged syntax diagnostics.
14#[derive(Debug, Clone)]
15pub struct ParseOutput {
16    pub cst: SyntaxNode,
17    pub diagnostics: Vec<ParseDiagnostic>,
18}
19
20impl ParseOutput {
21    /// Navigate the root through typed source wrappers.
22    pub fn document(&self) -> Document {
23        Document::cast(self.cst.clone()).expect("Rd parses always have a ROOT")
24    }
25}
26
27/// Parse a complete UTF-8 Rd document without expanding or evaluating macros.
28pub fn parse(text: &str) -> ParseOutput {
29    let mut parser = Parser {
30        text,
31        pos: 0,
32        builder: GreenNodeBuilder::new(),
33        diagnostics: Vec::new(),
34    };
35    parser.builder.start_node(K::ROOT.into());
36    parser.sequence(
37        Context {
38            mode: Mode::Prose,
39            items: Items::None,
40            sections: true,
41        },
42        Stop::Eof,
43        &mut State::default(),
44        0,
45    );
46    parser.builder.finish_node();
47    parser
48        .diagnostics
49        .sort_by_key(|diagnostic| (diagnostic.start, diagnostic.end));
50    ParseOutput {
51        cst: SyntaxNode::new_root(parser.builder.finish()),
52        diagnostics: parser.diagnostics,
53    }
54}
55
56/// Reconstruct every source byte, including malformed input and line endings.
57pub fn reconstruct(text: &str) -> String {
58    parse(text).cst.text().to_string()
59}
60
61#[derive(Clone, Copy)]
62struct Context {
63    mode: Mode,
64    items: Items,
65    sections: bool,
66}
67
68#[derive(Clone, Copy, PartialEq, Eq)]
69enum Stop {
70    Eof,
71    Brace,
72    Bracket,
73    Endif,
74}
75
76#[derive(Default)]
77struct State {
78    quote: Option<(u8, usize)>,
79    braces: usize,
80    r_comment: bool,
81}
82
83struct Parser<'a> {
84    text: &'a str,
85    pos: usize,
86    builder: GreenNodeBuilder<'static>,
87    diagnostics: Vec<ParseDiagnostic>,
88}
89
90impl Parser<'_> {
91    fn byte(&self) -> Option<u8> {
92        self.text.as_bytes().get(self.pos).copied()
93    }
94
95    fn emit(&mut self, kind: K, end: usize) {
96        if end > self.pos {
97            self.builder.token(kind.into(), &self.text[self.pos..end]);
98            self.pos = end;
99        }
100    }
101
102    fn diagnostic(&mut self, message: &str, start: usize, end: usize) {
103        self.diagnostics.push(ParseDiagnostic {
104            message: message.to_owned(),
105            start,
106            end,
107        });
108    }
109
110    fn line_end(&self) -> usize {
111        self.text[self.pos..]
112            .find('\n')
113            .map_or(self.text.len(), |offset| self.pos + offset)
114    }
115
116    fn directive(&self) -> Option<&'static str> {
117        if self.pos != 0 && self.text.as_bytes()[self.pos - 1] != b'\n' {
118            return None;
119        }
120        ["#ifdef", "#ifndef", "#endif"].into_iter().find(|name| {
121            self.text[self.pos..].starts_with(name)
122                && self
123                    .text
124                    .as_bytes()
125                    .get(self.pos + name.len())
126                    .is_none_or(|byte| !byte.is_ascii_alphabetic())
127        })
128    }
129
130    fn leaf(mode: Mode) -> K {
131        match mode {
132            Mode::Prose => K::TEXT,
133            Mode::RLike => K::RCODE,
134            Mode::Verbatim | Mode::Equation => K::VERBATIM,
135        }
136    }
137
138    fn sequence(&mut self, context: Context, stop: Stop, state: &mut State, depth: usize) {
139        // This bounds Rust and rowan recursion on hostile input while retaining
140        // a complete tree, rather than turning a resource limit into a panic.
141        if depth >= 128 {
142            self.diagnostic("Rd nesting limit exceeded", self.pos, self.pos);
143            self.emit(K::ERROR, self.text.len());
144            return;
145        }
146        while let Some(byte) = self.byte() {
147            if let Some(name) = self.directive().filter(|_| context.mode != Mode::Equation) {
148                if name == "#endif" {
149                    if stop == Stop::Endif {
150                        return;
151                    }
152                    self.diagnostic("unmatched #endif", self.pos, self.pos + name.len());
153                    self.emit(K::DIRECTIVE, self.line_end());
154                } else {
155                    self.conditional(context, state, depth + 1, name);
156                }
157                continue;
158            }
159            if byte == b'%' && context.mode != Mode::Equation && !state.r_comment {
160                let end = self.line_end();
161                let end = if self.text.as_bytes()[end - 1] == b'\r' {
162                    end - 1
163                } else {
164                    end
165                };
166                self.emit(K::COMMENT, end);
167                continue;
168            }
169            if byte == b'}' && state.quote.is_none() && state.braces == 0 {
170                if stop == Stop::Brace {
171                    return;
172                }
173                // An unclosed conditional must not consume its parent's brace.
174                if stop == Stop::Endif {
175                    return;
176                }
177                self.diagnostic("unexpected closing brace", self.pos, self.pos + 1);
178                self.emit(K::RBRACE, self.pos + 1);
179                continue;
180            }
181            if byte == b']' && stop == Stop::Bracket {
182                return;
183            }
184            if byte == b'{' && context.mode == Mode::Prose {
185                if context.sections {
186                    self.diagnostic(
187                        "unexpected brace group at document level",
188                        self.pos,
189                        self.pos + 1,
190                    );
191                }
192                self.group(K::GROUP, context, depth + 1);
193                continue;
194            }
195            if byte == b'\\' {
196                let next = self.text.as_bytes().get(self.pos + 1).copied();
197                let markup = match context.mode {
198                    Mode::Prose => next.is_some_and(|byte| byte.is_ascii_alphabetic()),
199                    Mode::RLike if !state.r_comment => next.is_some_and(|byte| {
200                        byte.is_ascii_alphabetic()
201                            && (state.quote.is_none() || matches!(byte, b'l' | b'v'))
202                    }),
203                    _ => false,
204                };
205                if markup {
206                    self.macro_call(context, depth + 1);
207                    continue;
208                }
209                if self.escape(context, state) {
210                    continue;
211                }
212            }
213            if context.mode == Mode::RLike
214                && state.quote.is_none()
215                && !state.r_comment
216                && matches!(byte, b'r' | b'R')
217                && self.raw_string()
218            {
219                continue;
220            }
221            self.text_run(context, stop, state);
222        }
223        if let Some((_, start)) = state.quote.take() {
224            self.diagnostic("unterminated R-like string", start, self.pos);
225        }
226    }
227
228    fn text_run(&mut self, context: Context, stop: Stop, state: &mut State) {
229        let start = self.pos;
230        let mut cursor = start;
231        let bytes = self.text.as_bytes();
232        while let Some(&byte) = bytes.get(cursor) {
233            if cursor > start
234                && (byte == b'\\'
235                    || (byte == b'%' && context.mode != Mode::Equation && !state.r_comment)
236                    || (byte == b'{' && context.mode == Mode::Prose)
237                    || (byte == b'}' && state.quote.is_none() && state.braces == 0)
238                    || (byte == b']' && stop == Stop::Bracket)
239                    || (byte == b'#' && bytes[cursor - 1] == b'\n')
240                    || (context.mode == Mode::RLike
241                        && state.quote.is_none()
242                        && !state.r_comment
243                        && matches!(byte, b'r' | b'R')
244                        && matches!(bytes.get(cursor + 1), Some(b'\'' | b'"'))))
245            {
246                break;
247            }
248            if context.mode == Mode::RLike {
249                if byte == b'\n' {
250                    state.r_comment = false;
251                }
252                if !state.r_comment {
253                    if let Some((quote, _)) = state.quote {
254                        if byte == quote {
255                            state.quote = None;
256                        }
257                    } else if matches!(byte, b'\'' | b'"' | b'`') {
258                        state.quote = Some((byte, cursor));
259                    } else if byte == b'#' {
260                        state.r_comment = true;
261                    }
262                }
263            }
264            if state.quote.is_none() && context.mode != Mode::Prose {
265                if byte == b'{' {
266                    state.braces += 1;
267                }
268                if byte == b'}' {
269                    state.braces = state.braces.saturating_sub(1);
270                }
271            }
272            cursor += 1;
273            // Physical newlines are a useful scanner boundary and keep raw
274            // string and conditional recognition independent of surrounding text.
275            if byte == b'\n' {
276                break;
277            }
278        }
279        self.emit(Self::leaf(context.mode), cursor);
280    }
281
282    fn escape(&mut self, context: Context, state: &mut State) -> bool {
283        let bytes = self.text.as_bytes();
284        let Some(&next) = bytes.get(self.pos + 1) else {
285            return false;
286        };
287        if context.mode == Mode::RLike
288            && let Some((quote, _)) = state.quote
289        {
290            let count = bytes[self.pos..]
291                .iter()
292                .take_while(|&&byte| byte == b'\\')
293                .count();
294            if bytes.get(self.pos + count) == Some(&quote) {
295                if count % 4 == 0 {
296                    state.quote = None;
297                }
298                self.emit(K::R_STRING_ESCAPE, self.pos + count + 1);
299                return true;
300            }
301            // Consume a whole run once; rescanning it for each pair would
302            // make long quoted backslash runs quadratic.
303            if count > 1 {
304                for _ in 0..count / 2 {
305                    self.emit(K::ESCAPED_RCODE, self.pos + 2);
306                }
307                return true;
308            }
309        }
310        let decoded = matches!(next, b'\\' | b'%')
311            || (matches!(next, b'{' | b'}')
312                && (context.mode != Mode::RLike || state.quote.is_none()));
313        if decoded {
314            let kind = match context.mode {
315                Mode::Prose => K::ESCAPED_TEXT,
316                Mode::RLike => K::ESCAPED_RCODE,
317                Mode::Verbatim => K::ESCAPED_VERBATIM,
318                Mode::Equation => K::VERBATIM,
319            };
320            self.emit(kind, self.pos + 2);
321            return true;
322        }
323        // Unrecognized escapes in R strings and verbatim arguments retain both
324        // characters. Consuming the pair also protects an escaped quote.
325        if context.mode != Mode::Prose {
326            let width = self.text[self.pos + 1..].chars().next().unwrap().len_utf8();
327            if next == b'\n' {
328                state.r_comment = false;
329            }
330            self.emit(Self::leaf(context.mode), self.pos + 1 + width);
331            return true;
332        }
333        false
334    }
335
336    fn raw_string(&mut self) -> bool {
337        let bytes = self.text.as_bytes();
338        let Some(&quote @ (b'\'' | b'"')) = bytes.get(self.pos + 1) else {
339            return false;
340        };
341        let mut opener = self.pos + 2;
342        while bytes.get(opener) == Some(&b'-') {
343            opener += 1;
344        }
345        let close = match bytes.get(opener) {
346            Some(b'(') => ')',
347            Some(b'[') => ']',
348            Some(b'{') => '}',
349            Some(b'|') => '|',
350            _ => return false,
351        };
352        let ending = format!(
353            "{close}{}{}",
354            "-".repeat(opener - self.pos - 2),
355            char::from(quote)
356        );
357        let end = if let Some(offset) = self.text[opener + 1..].find(&ending) {
358            opener + 1 + offset + ending.len()
359        } else {
360            self.diagnostic("unterminated R-like raw string", self.pos, self.text.len());
361            self.text.len()
362        };
363        self.emit(K::RCODE, end);
364        true
365    }
366
367    fn group(&mut self, kind: K, context: Context, depth: usize) {
368        let start = self.pos;
369        self.builder.start_node(kind.into());
370        self.emit(K::LBRACE, self.pos + 1);
371        self.sequence(context, Stop::Brace, &mut State::default(), depth);
372        if self.byte() == Some(b'}') {
373            self.emit(K::RBRACE, self.pos + 1);
374        } else {
375            self.diagnostic("unclosed brace", start, start + 1);
376        }
377        self.builder.finish_node();
378    }
379
380    fn macro_call(&mut self, context: Context, depth: usize) {
381        let start = self.pos;
382        let mut end = start + 1;
383        while self
384            .text
385            .as_bytes()
386            .get(end)
387            .is_some_and(u8::is_ascii_alphanumeric)
388        {
389            end += 1;
390        }
391        let mut spec = macros::lookup(&self.text[start + 1..end], context.items);
392        // R retries unknown alphanumeric names without a trailing digit run.
393        if spec.is_none() {
394            let trimmed =
395                self.text[start + 1..end].trim_end_matches(|ch: char| ch.is_ascii_digit());
396            if let Some(found) = macros::lookup(trimmed, context.items) {
397                end = start + 1 + trimmed.len();
398                spec = Some(found);
399            }
400        }
401        self.builder.start_node(K::MACRO.into());
402        self.emit(K::MACRO_NAME, end);
403        if let Some(spec) = spec {
404            if context.sections && !spec.top_level {
405                self.diagnostic("unexpected inline macro at document level", start, end);
406            }
407            if spec.section && !context.sections {
408                self.diagnostic(
409                    "unexpected section header inside an Rd argument",
410                    start,
411                    end,
412                );
413            }
414            if spec.option && self.byte() == Some(b'[') {
415                self.option(depth);
416            }
417            for (index, &mode) in spec.modes.iter().enumerate() {
418                if self.byte() == Some(b'{') {
419                    self.group(
420                        K::ARGUMENT,
421                        Context {
422                            mode,
423                            items: spec.items,
424                            sections: false,
425                        },
426                        depth,
427                    );
428                } else if index < spec.required {
429                    self.diagnostic("expected a braced macro argument", self.pos, self.pos);
430                    break;
431                }
432            }
433        }
434        self.builder.finish_node();
435    }
436
437    fn option(&mut self, depth: usize) {
438        let start = self.pos;
439        self.builder.start_node(K::OPTION.into());
440        self.emit(K::LBRACKET, self.pos + 1);
441        self.sequence(
442            Context {
443                mode: Mode::Prose,
444                items: Items::None,
445                sections: false,
446            },
447            Stop::Bracket,
448            &mut State::default(),
449            depth,
450        );
451        if self.byte() == Some(b']') {
452            self.emit(K::RBRACKET, self.pos + 1);
453        } else {
454            self.diagnostic("unclosed macro option", start, start + 1);
455        }
456        self.builder.finish_node();
457    }
458
459    fn conditional(&mut self, context: Context, state: &mut State, depth: usize, name: &str) {
460        let start = self.pos;
461        self.builder.start_node(K::CONDITIONAL.into());
462        self.emit(K::DIRECTIVE, self.pos + name.len());
463        let end = self.line_end();
464        if self.text[self.pos..end].trim().is_empty() {
465            self.diagnostic("expected a conditional target", self.pos, end);
466        }
467        self.emit(
468            K::TARGET,
469            (end + usize::from(end < self.text.len())).min(self.text.len()),
470        );
471        self.sequence(context, Stop::Endif, state, depth);
472        if self.directive() == Some("#endif") {
473            self.emit(K::DIRECTIVE, self.line_end());
474            if self.byte() == Some(b'\n') {
475                self.emit(K::DIRECTIVE, self.pos + 1);
476            }
477        } else {
478            self.diagnostic("unclosed Rd conditional", start, start + name.len());
479        }
480        self.builder.finish_node();
481    }
482}
arity_parser/rd/parser.rs

arity_parser/rd/
parser.rs
