arity_parser/dcf/
parser.rs

1//! The DCF parser: one forward pass over physical lines.
2//!
3//! Unlike the R grammar, this needs no event pipeline. Events exist there
4//! because the Pratt parser must retroactively wrap tokens it has already
5//! passed; DCF never backtracks and nests no deeper than
6//! `ROOT > RECORD > FIELD > VALUE_LINE`, so a direct [`GreenNodeBuilder`] walk
7//! is the whole parser.
8//!
9//! Losslessness is **structural**: the scanner carries one byte cursor and
10//! every emitted token is a slice `text[a..b]` with `a` the previous token's
11//! end. Note the deliberate absence of [`str::lines`] — it eats the `\r` of a
12//! CRLF pair and erases the difference between a final line with and without a
13//! trailing newline.
14
15use rowan::GreenNodeBuilder;
16
17use crate::dcf::ast::Document;
18use crate::dcf::syntax::{SyntaxKind, SyntaxNode};
19use crate::parser::diagnostics::push_diagnostic;
20
21pub use crate::parser::ParseDiagnostic;
22
23/// Reported for a line that is neither a field, a continuation, a comment, nor
24/// blank. R's `read.dcf` errors here ("Line starting ... is malformed!").
25pub(crate) const MALFORMED_LINE: &str =
26    "malformed line: expected 'Field: value' or an indented continuation line";
27
28/// Reported for an indented line with no field to continue. R's `read.dcf`
29/// errors here ("Found continuation line ... at begin of record.").
30pub(crate) const ORPHAN_CONTINUATION: &str = "continuation line at the start of a record";
31
32/// Reported for `: value` — a colon with nothing before it. R's `read.dcf`
33/// treats this as malformed too.
34pub(crate) const EMPTY_FIELD_NAME: &str = "empty field name";
35
36#[derive(Debug, Clone)]
37pub struct ParseOutput {
38    pub cst: SyntaxNode,
39    pub diagnostics: Vec<ParseDiagnostic>,
40}
41
42impl ParseOutput {
43    /// The typed view of the parsed document.
44    pub fn document(&self) -> Document {
45        use rowan::ast::AstNode as _;
46        Document::cast(self.cst.clone()).expect("the DCF parser always roots the tree at ROOT")
47    }
48}
49
50/// Parse DCF text into a lossless CST plus any diagnostics.
51///
52/// The parse never fails and never drops a byte: a malformed line is kept as a
53/// [`MALFORMED_LINE`](SyntaxKind::MALFORMED_LINE) node and reported on the
54/// side channel.
55pub fn parse(text: &str) -> ParseOutput {
56    Scanner::new(text).run()
57}
58
59/// Round-trip `text` through the parser. Always equal to `text`.
60pub fn reconstruct(text: &str) -> String {
61    parse(text)
62        .cst
63        .descendants_with_tokens()
64        .filter_map(|el| el.into_token())
65        .map(|tok| tok.text().to_string())
66        .collect::<String>()
67}
68
69struct Scanner<'a> {
70    text: &'a str,
71    builder: GreenNodeBuilder<'static>,
72    diagnostics: Vec<ParseDiagnostic>,
73    record_open: bool,
74    field_open: bool,
75}
76
77impl<'a> Scanner<'a> {
78    fn new(text: &'a str) -> Self {
79        Self {
80            text,
81            builder: GreenNodeBuilder::new(),
82            diagnostics: Vec::new(),
83            record_open: false,
84            field_open: false,
85        }
86    }
87
88    fn run(mut self) -> ParseOutput {
89        self.builder.start_node(SyntaxKind::ROOT.into());
90
91        let mut pos = 0;
92        while pos < self.text.len() {
93            pos = self.line(pos);
94        }
95
96        self.close_record();
97        self.builder.finish_node();
98
99        let green = self.builder.finish();
100        ParseOutput {
101            cst: SyntaxNode::new_root(green),
102            diagnostics: self.diagnostics,
103        }
104    }
105
106    // --- node stack -------------------------------------------------------
107
108    fn close_field(&mut self) {
109        if self.field_open {
110            self.builder.finish_node();
111            self.field_open = false;
112        }
113    }
114
115    fn close_record(&mut self) {
116        self.close_field();
117        if self.record_open {
118            self.builder.finish_node();
119            self.record_open = false;
120        }
121    }
122
123    fn open_record(&mut self) {
124        if !self.record_open {
125            self.builder.start_node(SyntaxKind::RECORD.into());
126            self.record_open = true;
127        }
128    }
129
130    fn token(&mut self, kind: SyntaxKind, start: usize, end: usize) {
131        // Empty tokens are a rowan footgun and would break the "every leaf is
132        // non-empty" invariant the tiling test asserts.
133        if start < end {
134            self.builder.token(kind.into(), &self.text[start..end]);
135        }
136    }
137
138    // --- lines ------------------------------------------------------------
139
140    /// Consume the physical line starting at `pos`; return the next position.
141    fn line(&mut self, pos: usize) -> usize {
142        let (content_end, line_end) = self.line_bounds(pos);
143        let content = &self.text[pos..content_end];
144
145        if content.trim().is_empty() {
146            // Empty *or* whitespace-only: a record separator, faithful to
147            // `read.dcf`, which starts a new record on either.
148            self.close_record();
149            self.builder.start_node(SyntaxKind::BLANK_LINE.into());
150            self.token(SyntaxKind::WHITESPACE, pos, content_end);
151            self.token(SyntaxKind::NEWLINE, content_end, line_end);
152            self.builder.finish_node();
153        } else if content.starts_with('#') {
154            // A comment at column zero attaches to whatever is open and closes
155            // nothing: `read.dcf` skips it and the enclosing field's
156            // continuation *resumes* on the next line. It never opens a record
157            // either, so a comment cannot bridge two records.
158            self.builder.start_node(SyntaxKind::COMMENT_LINE.into());
159            self.token(SyntaxKind::COMMENT, pos, content_end);
160            self.token(SyntaxKind::NEWLINE, content_end, line_end);
161            self.builder.finish_node();
162        } else if content.starts_with([' ', '\t']) {
163            if self.field_open {
164                self.value_line(pos, content_end, line_end);
165            } else {
166                push_diagnostic(&mut self.diagnostics, ORPHAN_CONTINUATION, pos, content_end);
167                self.open_record();
168                self.malformed_line(pos, content_end, line_end);
169            }
170        } else {
171            match content.find(':') {
172                Some(offset) => self.field(pos, pos + offset, content_end, line_end),
173                None => {
174                    push_diagnostic(&mut self.diagnostics, MALFORMED_LINE, pos, content_end);
175                    // A column-zero line ends any open field, malformed or not.
176                    self.close_field();
177                    self.open_record();
178                    self.malformed_line(pos, content_end, line_end);
179                }
180            }
181        }
182
183        line_end
184    }
185
186    /// `(content_end, line_end)` for the line starting at `pos`: the byte
187    /// offset where the line's content stops and where the line (newline
188    /// included) stops. Equal when the final line has no trailing newline.
189    fn line_bounds(&self, pos: usize) -> (usize, usize) {
190        match self.text[pos..].find('\n') {
191            Some(offset) => {
192                let line_end = pos + offset + 1;
193                let mut content_end = line_end - 1;
194                if content_end > pos && self.text.as_bytes()[content_end - 1] == b'\r' {
195                    content_end -= 1;
196                }
197                (content_end, line_end)
198            }
199            None => (self.text.len(), self.text.len()),
200        }
201    }
202
203    fn malformed_line(&mut self, start: usize, content_end: usize, line_end: usize) {
204        self.builder.start_node(SyntaxKind::MALFORMED_LINE.into());
205        self.token(SyntaxKind::TEXT, start, content_end);
206        self.token(SyntaxKind::NEWLINE, content_end, line_end);
207        self.builder.finish_node();
208    }
209
210    /// A field's first line: `Name`, optional whitespace, `:`, then the
211    /// remainder as this field's first [`VALUE_LINE`](SyntaxKind::VALUE_LINE).
212    fn field(&mut self, start: usize, colon: usize, content_end: usize, line_end: usize) {
213        let name = &self.text[start..colon];
214        let name_end = start + name.trim_end().len();
215        if name_end == start {
216            push_diagnostic(&mut self.diagnostics, EMPTY_FIELD_NAME, colon, colon + 1);
217        }
218
219        self.close_field();
220        self.open_record();
221        self.builder.start_node(SyntaxKind::FIELD.into());
222        self.field_open = true;
223
224        self.token(SyntaxKind::FIELD_NAME, start, name_end);
225        // Whitespace between the name and the colon (`Package : p`) must be its
226        // own token: a `FIELD_NAME` that swallowed it would still satisfy
227        // `starts_with("Collate")` while silently breaking `== "Package"`.
228        self.token(SyntaxKind::WHITESPACE, name_end, colon);
229        self.token(SyntaxKind::COLON, colon, colon + 1);
230
231        self.value_line(colon + 1, content_end, line_end);
232    }
233
234    /// One value line, spanning `start..line_end`. Emits nothing at all when
235    /// the span is empty — that is `Package:` at EOF, a field with no value.
236    fn value_line(&mut self, start: usize, content_end: usize, line_end: usize) {
237        if start == line_end {
238            return;
239        }
240        let slice = &self.text[start..content_end];
241        let trimmed = slice.trim();
242        // An all-whitespace value is one leading-whitespace run, not a leading
243        // and a trailing run — computing the two independently would overlap
244        // them and emit those bytes twice.
245        let (lead, trail) = if trimmed.is_empty() {
246            (content_end, content_end)
247        } else {
248            let lead = start + (slice.len() - slice.trim_start().len());
249            (lead, lead + trimmed.len())
250        };
251
252        self.builder.start_node(SyntaxKind::VALUE_LINE.into());
253        self.token(SyntaxKind::WHITESPACE, start, lead);
254        self.token(SyntaxKind::VALUE_TEXT, lead, trail);
255        self.token(SyntaxKind::WHITESPACE, trail, content_end);
256        self.token(SyntaxKind::NEWLINE, content_end, line_end);
257        self.builder.finish_node();
258    }
259}
260
261#[cfg(test)]
262mod tests {
263    use super::*;
264    use crate::dcf::syntax::SyntaxKind;
265
266    fn doc(text: &str) -> ParseOutput {
267        parse(text)
268    }
269
270    fn folded(text: &str, name: &str) -> Option<String> {
271        doc(text).document().field(name).map(|f| f.folded_value())
272    }
273
274    /// Every structural guarantee the tree owes its consumers, checked at once:
275    /// no empty leaves, the leaves tile the source exactly, every newline lands
276    /// in a line node, and no line node holds more than one.
277    fn assert_tiles(text: &str) {
278        let output = parse(text);
279        let mut rebuilt = String::new();
280        let mut newlines = 0;
281
282        for token in output
283            .cst
284            .descendants_with_tokens()
285            .filter_map(|el| el.into_token())
286        {
287            assert!(
288                !token.text().is_empty(),
289                "empty {:?} token in {text:?}",
290                token.kind()
291            );
292            rebuilt.push_str(token.text());
293
294            if token.kind() == SyntaxKind::NEWLINE {
295                newlines += 1;
296                let parent = token.parent().expect("a token always has a parent");
297                assert!(
298                    parent.kind().is_line(),
299                    "NEWLINE parented by {:?}, not a line node, in {text:?}",
300                    parent.kind()
301                );
302            }
303        }
304
305        assert_eq!(rebuilt, text, "leaves do not tile the source");
306        assert_eq!(
307            newlines,
308            text.matches('\n').count(),
309            "newline count mismatch in {text:?}"
310        );
311
312        for node in output.cst.descendants().filter(|n| n.kind().is_line()) {
313            let count = node
314                .children_with_tokens()
315                .filter_map(|el| el.into_token())
316                .filter(|t| t.kind() == SyntaxKind::NEWLINE)
317                .count();
318            assert!(
319                count <= 1,
320                "{:?} holds {count} newlines in {text:?}",
321                node.kind()
322            );
323        }
324    }
325
326    /// Adversarial inputs, every one of which must round-trip and tile. This is
327    /// the crate's losslessness bar (Tenet 4), stated as a table rather than a
328    /// property crate so the parser crate stays dependency-thin.
329    const ADVERSARIAL: &[&str] = &[
330        "",
331        "\n",
332        "\r\n",
333        "\n\n\n",
334        "Package: p",
335        "Package: p\n",
336        "Package:\n",
337        "Package:",
338        "Package: ",
339        ":",
340        ": v\n",
341        "   \n",
342        "\t\n",
343        "   ",
344        "Package: p\r\nVersion: 1\r\n",
345        "Package: p\r\n  cont\r\n",
346        "a\rb\n",
347        "\u{FEFF}Package: p\n",
348        "# c",
349        "# c\n",
350        "#\n",
351        "   # c\n",
352        "Collate:\n a.R\n# c\n b.R\n",
353        "garbage\n",
354        "garbage",
355        "  orphan\n",
356        "Package: p\n\n  orphan\n",
357        "Package: p\n\nVersion: 1\n",
358        "Date/Publication: 2025-09-12 07:20:14 UTC\n",
359        "Authors@R: c(person(\"A\", \"B\", role = c(\"aut\", \"cre\")))\n",
360        "Collate:\n    'a.R'\n    'b.R'\n",
361        "Package: p\n   \nVersion: 1\n",
362        "Package : p\n",
363        "Built: R 4.5.3; ; 2025-01-01 00:00:00 UTC; unix\n",
364    ];
365
366    #[test]
367    fn round_trips_every_adversarial_input() {
368        for input in ADVERSARIAL {
369            assert_eq!(
370                &reconstruct(input),
371                input,
372                "round-trip failed for {input:?}"
373            );
374            assert_tiles(input);
375        }
376    }
377
378    #[test]
379    fn folds_continuation_lines() {
380        assert_eq!(folded("Package: p\n", "Package").as_deref(), Some("p"));
381        assert_eq!(
382            folded("Depends:\n    a,\n    b\n", "Depends").as_deref(),
383            Some("\na,\nb")
384        );
385    }
386
387    /// A field whose own line is empty folds with a leading `\n`. R's
388    /// `read.dcf` yields `"a.R\nb.R"` here; arity keeps the empty leading
389    /// segment because every caller predating this parser did.
390    #[test]
391    fn folded_value_keeps_leading_newline_artifact() {
392        let text = "Package: testpkg\nCollate:\n    a.R\n    b.R\nVersion: 1.0\n";
393        assert_eq!(folded(text, "Collate").as_deref(), Some("\na.R\nb.R"));
394        assert_eq!(folded(text, "Package").as_deref(), Some("testpkg"));
395        assert_eq!(folded(text, "Version").as_deref(), Some("1.0"));
396    }
397
398    /// The `Roxygen` field is fed straight to the *R* parser downstream, so the
399    /// folded text has to stay valid R. This crate owns both grammars, so it
400    /// can assert the cross-grammar contract itself.
401    #[test]
402    fn folded_roxygen_field_parses_as_r() {
403        let text = "Package: p\nRoxygen: list(load = \"installed\",\n    markdown = TRUE)\n";
404        let value = folded(text, "Roxygen").expect("Roxygen field");
405        assert_eq!(value, "list(load = \"installed\",\nmarkdown = TRUE)");
406        let r = crate::parser::parse(&value);
407        assert!(r.diagnostics.is_empty(), "folded value is not valid R");
408    }
409
410    /// The migration's load-bearing case: a blank line splits records, but the
411    /// `DESCRIPTION` readers are record-blind and must still see every field.
412    #[test]
413    fn blank_line_starts_a_new_record_without_hiding_fields() {
414        let output = doc("Package: p\n\nVersion: 1\n");
415        assert_eq!(output.document().records().count(), 2);
416        assert_eq!(
417            folded("Package: p\n\nVersion: 1\n", "Version").as_deref(),
418            Some("1")
419        );
420    }
421
422    /// A whitespace-only line is a separator, not a continuation. Before this
423    /// parser it appended a bare `"\n"` to the preceding value — and that value
424    /// is spliced into a filesystem path by the harvester.
425    #[test]
426    fn whitespace_only_line_is_blank_not_a_continuation() {
427        let text = "Package: p\n   \nVersion: 1\n";
428        assert_eq!(folded(text, "Package").as_deref(), Some("p"));
429        assert_eq!(doc(text).document().records().count(), 2);
430    }
431
432    #[test]
433    fn comment_at_column_zero_is_skipped() {
434        let text = "Package: p\n# a comment\nVersion: 1\n";
435        assert_eq!(doc(text).document().fields().count(), 2);
436        assert_eq!(folded(text, "Version").as_deref(), Some("1"));
437    }
438
439    /// A `#` at column zero inside a continuation block does not end the field:
440    /// `read.dcf` skips the line and the continuation resumes.
441    #[test]
442    fn comment_between_continuations_keeps_the_field() {
443        let text = "Collate:\n a.R\n# skip me\n b.R\n";
444        assert_eq!(folded(text, "Collate").as_deref(), Some("\na.R\nb.R"));
445        assert_eq!(doc(text).document().fields().count(), 1);
446    }
447
448    #[test]
449    fn indented_hash_is_a_continuation() {
450        let text = "Description: one\n  # two\n";
451        assert_eq!(folded(text, "Description").as_deref(), Some("one\n# two"));
452    }
453
454    /// A commented-out field is a comment, not a field named `# Depends`.
455    #[test]
456    fn commented_field_is_not_a_field() {
457        let text = "Package: p\n# Depends: R (>= 4.0)\n";
458        assert!(folded(text, "Depends").is_none());
459        assert!(folded(text, "# Depends").is_none());
460    }
461
462    #[test]
463    fn malformed_line_is_diagnosed_and_preserved() {
464        let output = doc("Package: p\ngarbage\nVersion: 1\n");
465        assert_eq!(output.diagnostics.len(), 1);
466        assert_eq!(output.diagnostics[0].message, MALFORMED_LINE);
467        assert_eq!(
468            &"Package: p\ngarbage\nVersion: 1\n"
469                [output.diagnostics[0].start..output.diagnostics[0].end],
470            "garbage"
471        );
472        // The surrounding fields are unaffected.
473        assert_eq!(output.document().fields().count(), 2);
474    }
475
476    #[test]
477    fn orphan_continuation_is_diagnosed() {
478        let output = doc("Package: p\n\n  orphan\n");
479        assert_eq!(output.diagnostics.len(), 1);
480        assert_eq!(output.diagnostics[0].message, ORPHAN_CONTINUATION);
481        // It does not silently join the previous record's last field.
482        assert_eq!(
483            folded("Package: p\n\n  orphan\n", "Package").as_deref(),
484            Some("p")
485        );
486    }
487
488    #[test]
489    fn empty_field_name_is_diagnosed() {
490        let output = doc(": value\n");
491        assert_eq!(output.diagnostics.len(), 1);
492        assert_eq!(output.diagnostics[0].message, EMPTY_FIELD_NAME);
493        let field = output.document().fields().next().expect("a field");
494        assert_eq!(field.name(), "");
495        assert_eq!(field.folded_value(), "value");
496    }
497
498    /// arity resolves a duplicate to the **first** occurrence; R's `read.dcf`
499    /// takes the last. Deliberate, and pinned so the divergence cannot drift.
500    #[test]
501    fn first_duplicate_wins() {
502        assert_eq!(
503            folded("Package: first\nPackage: second\n", "Package").as_deref(),
504            Some("first")
505        );
506    }
507
508    /// arity trims the name. R's `read.dcf` does **not**: `Package : p`
509    /// declares a field literally named `"Package "`, so R sees no `Package`
510    /// at all. Trimming is the lenient direction, and the whitespace survives
511    /// as its own token, so a lint can still flag the typo precisely.
512    #[test]
513    fn field_name_excludes_trailing_space() {
514        let text = "Package : p\n";
515        let field = doc(text).document().fields().next().expect("a field");
516        assert_eq!(field.name(), "Package");
517        assert_eq!(field.folded_value(), "p");
518    }
519
520    /// `folded_value` would *hide* a stray `\r` (`str::trim` eats it), so CRLF
521    /// correctness has to be asserted on the tree.
522    #[test]
523    fn crlf_newline_is_one_token() {
524        let output = doc("Package: p\r\n  cont\r\n");
525        let newlines: Vec<String> = output
526            .cst
527            .descendants_with_tokens()
528            .filter_map(|el| el.into_token())
529            .filter(|t| t.kind() == SyntaxKind::NEWLINE)
530            .map(|t| t.text().to_string())
531            .collect();
532        assert_eq!(newlines, vec!["\r\n", "\r\n"]);
533
534        for token in output
535            .cst
536            .descendants_with_tokens()
537            .filter_map(|el| el.into_token())
538            .filter(|t| t.kind() == SyntaxKind::VALUE_TEXT)
539        {
540            assert!(!token.text().contains('\r'), "\\r leaked into VALUE_TEXT");
541        }
542    }
543
544    #[test]
545    fn value_keeps_colons_after_the_first() {
546        assert_eq!(
547            folded("Built: R 4.5.3; ; 2025-01-01 00:00:00 UTC; unix\n", "Built").as_deref(),
548            Some("R 4.5.3; ; 2025-01-01 00:00:00 UTC; unix")
549        );
550        assert_eq!(
551            folded(
552                "Date/Publication: 2025-09-12 07:20:14 UTC\n",
553                "Date/Publication"
554            )
555            .as_deref(),
556            Some("2025-09-12 07:20:14 UTC")
557        );
558    }
559
560    /// A field with no value at all emits no value line, and folds to `""`.
561    #[test]
562    fn field_without_a_value() {
563        let field = doc("Package:").document().fields().next().expect("a field");
564        assert_eq!(field.value_lines().count(), 0);
565        assert_eq!(field.folded_value(), "");
566    }
567
568    #[test]
569    fn raw_value_text_keeps_every_byte_after_the_colon() {
570        let field = doc("Collate:\n    a.R\n    b.R\n")
571            .document()
572            .fields()
573            .next()
574            .expect("a field");
575        assert_eq!(field.raw_value_text(), "\n    a.R\n    b.R\n");
576    }
577
578    #[test]
579    fn clean_input_reports_no_diagnostics() {
580        for input in [
581            "Package: p\n",
582            "Package: p\n\nVersion: 1\n",
583            "Collate:\n a.R\n# c\n b.R\n",
584            "   \n",
585            "# only a comment\n",
586        ] {
587            assert!(
588                doc(input).diagnostics.is_empty(),
589                "unexpected diagnostics for {input:?}: {:?}",
590                doc(input).diagnostics
591            );
592        }
593    }
594}
arity_parser/dcf/parser.rs

arity_parser/dcf/
parser.rs
