arity_parser/dcf/
parser.rs

1//! The DCF parser: one forward pass over physical lines.
2//!
3//! Unlike the R grammar, this needs no event pipeline. Events exist there
4//! because the Pratt parser must retroactively wrap tokens it has already
5//! passed; DCF never backtracks and nests no deeper than
6//! `ROOT > RECORD > FIELD > VALUE_LINE`, so a direct [`GreenNodeBuilder`] walk
7//! is the whole parser.
8//!
9//! Losslessness is **structural**: the scanner carries one byte cursor and
10//! every emitted token is a slice `text[a..b]` with `a` the previous token's
11//! end. Note the deliberate absence of [`str::lines`] — it eats the `\r` of a
12//! CRLF pair and erases the difference between a final line with and without a
13//! trailing newline.
14
15use rowan::GreenNodeBuilder;
16
17use crate::dcf::ast::Document;
18use crate::dcf::syntax::{SyntaxKind, SyntaxNode};
19use crate::parser::diagnostics::push_diagnostic;
20
21pub use crate::parser::ParseDiagnostic;
22
23/// Reported for a line that is neither a field, a continuation, a comment, nor
24/// blank. R's `read.dcf` errors here ("Line starting ... is malformed!").
25pub(crate) const MALFORMED_LINE: &str =
26    "malformed line: expected 'Field: value' or an indented continuation line";
27
28/// Reported for an indented line with no field to continue. R's `read.dcf`
29/// errors here ("Found continuation line ... at begin of record.").
30pub(crate) const ORPHAN_CONTINUATION: &str = "continuation line at the start of a record";
31
32/// Reported for `: value` — a colon with nothing before it. R's `read.dcf`
33/// treats this as malformed too.
34pub(crate) const EMPTY_FIELD_NAME: &str = "empty field name";
35
36#[derive(Debug, Clone)]
37pub struct ParseOutput {
38    pub cst: SyntaxNode,
39    pub diagnostics: Vec<ParseDiagnostic>,
40}
41
42impl ParseOutput {
43    /// The typed view of the parsed document.
44    pub fn document(&self) -> Document {
45        use rowan::ast::AstNode as _;
46        Document::cast(self.cst.clone()).expect("the DCF parser always roots the tree at ROOT")
47    }
48}
49
50/// Parse DCF text into a lossless CST plus any diagnostics.
51///
52/// The parse never fails and never drops a byte: a malformed line is kept as a
53/// [`MALFORMED_LINE`](SyntaxKind::MALFORMED_LINE) node and reported on the
54/// side channel.
55pub fn parse(text: &str) -> ParseOutput {
56    Scanner::new(text).run()
57}
58
59/// Round-trip `text` through the parser. Always equal to `text`.
60pub fn reconstruct(text: &str) -> String {
61    parse(text)
62        .cst
63        .descendants_with_tokens()
64        .filter_map(|el| el.into_token())
65        .map(|tok| tok.text().to_string())
66        .collect::<String>()
67}
68
69struct Scanner<'a> {
70    text: &'a str,
71    builder: GreenNodeBuilder<'static>,
72    diagnostics: Vec<ParseDiagnostic>,
73    record_open: bool,
74    field_open: bool,
75}
76
77impl<'a> Scanner<'a> {
78    fn new(text: &'a str) -> Self {
79        Self {
80            text,
81            builder: GreenNodeBuilder::new(),
82            diagnostics: Vec::new(),
83            record_open: false,
84            field_open: false,
85        }
86    }
87
88    fn run(mut self) -> ParseOutput {
89        self.builder.start_node(SyntaxKind::ROOT.into());
90
91        let mut pos = 0;
92        while pos < self.text.len() {
93            pos = self.line(pos);
94        }
95
96        self.close_record();
97        self.builder.finish_node();
98
99        let green = self.builder.finish();
100        ParseOutput {
101            cst: SyntaxNode::new_root(green),
102            diagnostics: self.diagnostics,
103        }
104    }
105
106    // --- node stack -------------------------------------------------------
107
108    fn close_field(&mut self) {
109        if self.field_open {
110            self.builder.finish_node();
111            self.field_open = false;
112        }
113    }
114
115    fn close_record(&mut self) {
116        self.close_field();
117        if self.record_open {
118            self.builder.finish_node();
119            self.record_open = false;
120        }
121    }
122
123    fn open_record(&mut self) {
124        if !self.record_open {
125            self.builder.start_node(SyntaxKind::RECORD.into());
126            self.record_open = true;
127        }
128    }
129
130    fn token(&mut self, kind: SyntaxKind, start: usize, end: usize) {
131        // Empty tokens are a rowan footgun and would break the "every leaf is
132        // non-empty" invariant the tiling test asserts.
133        if start < end {
134            self.builder.token(kind.into(), &self.text[start..end]);
135        }
136    }
137
138    // --- lines ------------------------------------------------------------
139
140    /// Consume the physical line starting at `pos`; return the next position.
141    fn line(&mut self, pos: usize) -> usize {
142        let (content_end, line_end) = self.line_bounds(pos);
143        let content = &self.text[pos..content_end];
144
145        if content.trim().is_empty() {
146            // Empty *or* whitespace-only: a record separator, faithful to
147            // `read.dcf`, which starts a new record on either.
148            self.close_record();
149            self.builder.start_node(SyntaxKind::BLANK_LINE.into());
150            self.token(SyntaxKind::WHITESPACE, pos, content_end);
151            self.token(SyntaxKind::NEWLINE, content_end, line_end);
152            self.builder.finish_node();
153        } else if content.starts_with('#') {
154            // A comment at column zero attaches to whatever is open and closes
155            // nothing: `read.dcf` skips it and the enclosing field's
156            // continuation *resumes* on the next line. It never opens a record
157            // either, so a comment cannot bridge two records.
158            self.builder.start_node(SyntaxKind::COMMENT_LINE.into());
159            self.token(SyntaxKind::COMMENT, pos, content_end);
160            self.token(SyntaxKind::NEWLINE, content_end, line_end);
161            self.builder.finish_node();
162        } else if content.starts_with([' ', '\t']) {
163            if self.field_open {
164                self.value_line(pos, content_end, line_end);
165            } else {
166                push_diagnostic(&mut self.diagnostics, ORPHAN_CONTINUATION, pos, content_end);
167                self.open_record();
168                self.malformed_line(pos, content_end, line_end);
169            }
170        } else {
171            match content.find(':') {
172                Some(offset) => self.field(pos, pos + offset, content_end, line_end),
173                None => {
174                    push_diagnostic(&mut self.diagnostics, MALFORMED_LINE, pos, content_end);
175                    // A column-zero line ends any open field, malformed or not.
176                    self.close_field();
177                    self.open_record();
178                    self.malformed_line(pos, content_end, line_end);
179                }
180            }
181        }
182
183        line_end
184    }
185
186    /// `(content_end, line_end)` for the line starting at `pos`: the byte
187    /// offset where the line's content stops and where the line (newline
188    /// included) stops. Equal when the final line has no trailing newline.
189    fn line_bounds(&self, pos: usize) -> (usize, usize) {
190        match self.text[pos..].find('\n') {
191            Some(offset) => {
192                let line_end = pos + offset + 1;
193                let mut content_end = line_end - 1;
194                if content_end > pos && self.text.as_bytes()[content_end - 1] == b'\r' {
195                    content_end -= 1;
196                }
197                (content_end, line_end)
198            }
199            None => (self.text.len(), self.text.len()),
200        }
201    }
202
203    fn malformed_line(&mut self, start: usize, content_end: usize, line_end: usize) {
204        self.builder.start_node(SyntaxKind::MALFORMED_LINE.into());
205        self.token(SyntaxKind::TEXT, start, content_end);
206        self.token(SyntaxKind::NEWLINE, content_end, line_end);
207        self.builder.finish_node();
208    }
209
210    /// A field's first line: `Name`, optional whitespace, `:`, then the
211    /// remainder as this field's first [`VALUE_LINE`](SyntaxKind::VALUE_LINE).
212    fn field(&mut self, start: usize, colon: usize, content_end: usize, line_end: usize) {
213        let name = &self.text[start..colon];
214        let name_end = start + name.trim_end().len();
215        if name_end == start {
216            push_diagnostic(&mut self.diagnostics, EMPTY_FIELD_NAME, colon, colon + 1);
217        }
218
219        self.close_field();
220        self.open_record();
221        self.builder.start_node(SyntaxKind::FIELD.into());
222        self.field_open = true;
223
224        self.token(SyntaxKind::FIELD_NAME, start, name_end);
225        // Whitespace between the name and the colon (`Package : p`) must be its
226        // own token: a `FIELD_NAME` that swallowed it would still satisfy
227        // `starts_with("Collate")` while silently breaking `== "Package"`.
228        self.token(SyntaxKind::WHITESPACE, name_end, colon);
229        self.token(SyntaxKind::COLON, colon, colon + 1);
230
231        self.value_line(colon + 1, content_end, line_end);
232    }
233
234    /// One value line, spanning `start..line_end`. Emits nothing at all when
235    /// the span is empty — that is `Package:` at EOF, a field with no value.
236    fn value_line(&mut self, start: usize, content_end: usize, line_end: usize) {
237        if start == line_end {
238            return;
239        }
240        let slice = &self.text[start..content_end];
241        let trimmed = slice.trim();
242        // An all-whitespace value is one leading-whitespace run, not a leading
243        // and a trailing run — computing the two independently would overlap
244        // them and emit those bytes twice.
245        let (lead, trail) = if trimmed.is_empty() {
246            (content_end, content_end)
247        } else {
248            let lead = start + (slice.len() - slice.trim_start().len());
249            (lead, lead + trimmed.len())
250        };
251
252        self.builder.start_node(SyntaxKind::VALUE_LINE.into());
253        self.token(SyntaxKind::WHITESPACE, start, lead);
254        self.token(SyntaxKind::VALUE_TEXT, lead, trail);
255        self.token(SyntaxKind::WHITESPACE, trail, content_end);
256        self.token(SyntaxKind::NEWLINE, content_end, line_end);
257        self.builder.finish_node();
258    }
259}
260
261#[cfg(test)]
262mod tests {
263    use super::*;
264    use crate::dcf::syntax::SyntaxKind;
265
266    fn doc(text: &str) -> ParseOutput {
267        parse(text)
268    }
269
270    fn folded(text: &str, name: &str) -> Option<String> {
271        doc(text).document().field(name).map(|f| f.folded_value())
272    }
273
274    /// Every structural guarantee the tree owes its consumers, checked at once:
275    /// no empty leaves, the leaves tile the source exactly, every newline lands
276    /// in a line node, and no line node holds more than one.
277    fn assert_tiles(text: &str) {
278        let output = parse(text);
279        let mut rebuilt = String::new();
280        let mut newlines = 0;
281
282        for token in output
283            .cst
284            .descendants_with_tokens()
285            .filter_map(|el| el.into_token())
286        {
287            assert!(
288                !token.text().is_empty(),
289                "empty {:?} token in {text:?}",
290                token.kind()
291            );
292            rebuilt.push_str(token.text());
293
294            if token.kind() == SyntaxKind::NEWLINE {
295                newlines += 1;
296                let parent = token.parent().expect("a token always has a parent");
297                assert!(
298                    parent.kind().is_line(),
299                    "NEWLINE parented by {:?}, not a line node, in {text:?}",
300                    parent.kind()
301                );
302            }
303        }
304
305        assert_eq!(rebuilt, text, "leaves do not tile the source");
306        assert_eq!(
307            newlines,
308            text.matches('\n').count(),
309            "newline count mismatch in {text:?}"
310        );
311
312        for node in output.cst.descendants().filter(|n| n.kind().is_line()) {
313            let count = node
314                .children_with_tokens()
315                .filter_map(|el| el.into_token())
316                .filter(|t| t.kind() == SyntaxKind::NEWLINE)
317                .count();
318            assert!(
319                count <= 1,
320                "{:?} holds {count} newlines in {text:?}",
321                node.kind()
322            );
323        }
324    }
325
326    /// Adversarial inputs, every one of which must round-trip and tile. This is
327    /// the crate's losslessness bar (Tenet 4), stated as a table rather than a
328    /// property crate so the parser crate stays dependency-thin.
329    const ADVERSARIAL: &[&str] = &[
330        "",
331        "\n",
332        "\r\n",
333        "\n\n\n",
334        "Package: p",
335        "Package: p\n",
336        "Package:\n",
337        "Package:",
338        "Package: ",
339        ":",
340        ": v\n",
341        "   \n",
342        "\t\n",
343        "   ",
344        "Package: p\r\nVersion: 1\r\n",
345        "Package: p\r\n  cont\r\n",
346        "a\rb\n",
347        "\u{FEFF}Package: p\n",
348        "# c",
349        "# c\n",
350        "#\n",
351        "   # c\n",
352        "Collate:\n a.R\n# c\n b.R\n",
353        "garbage\n",
354        "garbage",
355        "  orphan\n",
356        "Package: p\n\n  orphan\n",
357        "Package: p\n\nVersion: 1\n",
358        "Date/Publication: 2025-09-12 07:20:14 UTC\n",
359        "Authors@R: c(person(\"A\", \"B\", role = c(\"aut\", \"cre\")))\n",
360        "Collate:\n    'a.R'\n    'b.R'\n",
361        "Package: p\n   \nVersion: 1\n",
362        "Package : p\n",
363        "Built: R 4.5.3; ; 2025-01-01 00:00:00 UTC; unix\n",
364    ];
365
366    #[test]
367    fn round_trips_every_adversarial_input() {
368        for input in ADVERSARIAL {
369            assert_eq!(
370                &reconstruct(input),
371                input,
372                "round-trip failed for {input:?}"
373            );
374            assert_tiles(input);
375        }
376    }
377
378    #[test]
379    fn folds_continuation_lines() {
380        assert_eq!(folded("Package: p\n", "Package").as_deref(), Some("p"));
381        assert_eq!(
382            folded("Depends:\n    a,\n    b\n", "Depends").as_deref(),
383            Some("a,\nb")
384        );
385    }
386
387    /// A field whose own line is empty folds without a leading `\n`, matching
388    /// R's `read.dcf`.
389    #[test]
390    fn folded_value_drops_empty_first_line() {
391        let text = "Package: testpkg\nCollate:\n    a.R\n    b.R\nVersion: 1.0\n";
392        assert_eq!(folded(text, "Collate").as_deref(), Some("a.R\nb.R"));
393        assert_eq!(folded(text, "Package").as_deref(), Some("testpkg"));
394        assert_eq!(folded(text, "Version").as_deref(), Some("1.0"));
395    }
396
397    /// The `Roxygen` field is fed straight to the *R* parser downstream, so the
398    /// folded text has to stay valid R. This crate owns both grammars, so it
399    /// can assert the cross-grammar contract itself.
400    #[test]
401    fn folded_roxygen_field_parses_as_r() {
402        let text = "Package: p\nRoxygen: list(load = \"installed\",\n    markdown = TRUE)\n";
403        let value = folded(text, "Roxygen").expect("Roxygen field");
404        assert_eq!(value, "list(load = \"installed\",\nmarkdown = TRUE)");
405        let r = crate::parser::parse(&value);
406        assert!(r.diagnostics.is_empty(), "folded value is not valid R");
407    }
408
409    /// The migration's load-bearing case: a blank line splits records, but the
410    /// `DESCRIPTION` readers are record-blind and must still see every field.
411    #[test]
412    fn blank_line_starts_a_new_record_without_hiding_fields() {
413        let output = doc("Package: p\n\nVersion: 1\n");
414        assert_eq!(output.document().records().count(), 2);
415        assert_eq!(
416            folded("Package: p\n\nVersion: 1\n", "Version").as_deref(),
417            Some("1")
418        );
419    }
420
421    /// A whitespace-only line is a separator, not a continuation. Before this
422    /// parser it appended a bare `"\n"` to the preceding value — and that value
423    /// is spliced into a filesystem path by the harvester.
424    #[test]
425    fn whitespace_only_line_is_blank_not_a_continuation() {
426        let text = "Package: p\n   \nVersion: 1\n";
427        assert_eq!(folded(text, "Package").as_deref(), Some("p"));
428        assert_eq!(doc(text).document().records().count(), 2);
429    }
430
431    #[test]
432    fn comment_at_column_zero_is_skipped() {
433        let text = "Package: p\n# a comment\nVersion: 1\n";
434        assert_eq!(doc(text).document().fields().count(), 2);
435        assert_eq!(folded(text, "Version").as_deref(), Some("1"));
436    }
437
438    /// A `#` at column zero inside a continuation block does not end the field:
439    /// `read.dcf` skips the line and the continuation resumes.
440    #[test]
441    fn comment_between_continuations_keeps_the_field() {
442        let text = "Collate:\n a.R\n# skip me\n b.R\n";
443        assert_eq!(folded(text, "Collate").as_deref(), Some("a.R\nb.R"));
444        assert_eq!(doc(text).document().fields().count(), 1);
445    }
446
447    #[test]
448    fn indented_hash_is_a_continuation() {
449        let text = "Description: one\n  # two\n";
450        assert_eq!(folded(text, "Description").as_deref(), Some("one\n# two"));
451    }
452
453    /// A commented-out field is a comment, not a field named `# Depends`.
454    #[test]
455    fn commented_field_is_not_a_field() {
456        let text = "Package: p\n# Depends: R (>= 4.0)\n";
457        assert!(folded(text, "Depends").is_none());
458        assert!(folded(text, "# Depends").is_none());
459    }
460
461    #[test]
462    fn malformed_line_is_diagnosed_and_preserved() {
463        let output = doc("Package: p\ngarbage\nVersion: 1\n");
464        assert_eq!(output.diagnostics.len(), 1);
465        assert_eq!(output.diagnostics[0].message, MALFORMED_LINE);
466        assert_eq!(
467            &"Package: p\ngarbage\nVersion: 1\n"
468                [output.diagnostics[0].start..output.diagnostics[0].end],
469            "garbage"
470        );
471        // The surrounding fields are unaffected.
472        assert_eq!(output.document().fields().count(), 2);
473    }
474
475    #[test]
476    fn orphan_continuation_is_diagnosed() {
477        let output = doc("Package: p\n\n  orphan\n");
478        assert_eq!(output.diagnostics.len(), 1);
479        assert_eq!(output.diagnostics[0].message, ORPHAN_CONTINUATION);
480        // It does not silently join the previous record's last field.
481        assert_eq!(
482            folded("Package: p\n\n  orphan\n", "Package").as_deref(),
483            Some("p")
484        );
485    }
486
487    #[test]
488    fn empty_field_name_is_diagnosed() {
489        let output = doc(": value\n");
490        assert_eq!(output.diagnostics.len(), 1);
491        assert_eq!(output.diagnostics[0].message, EMPTY_FIELD_NAME);
492        let field = output.document().fields().next().expect("a field");
493        assert_eq!(field.name(), "");
494        assert_eq!(field.folded_value(), "value");
495    }
496
497    /// Like `read.dcf`, document-wide lookup resolves a duplicate to its last
498    /// occurrence, including when a blank line put it in a later record.
499    #[test]
500    fn last_duplicate_wins() {
501        assert_eq!(
502            folded("Package: first\n\nPackage: second\n", "Package").as_deref(),
503            Some("second")
504        );
505    }
506
507    /// Record-local lookup follows the same last-wins rule.
508    #[test]
509    fn last_duplicate_in_record_wins() {
510        let output = doc("Package: first\nPackage: second\n");
511        let record = output.document().first_record().expect("a record");
512        assert_eq!(
513            record.field("Package").map(|field| field.folded_value()),
514            Some("second".to_string())
515        );
516    }
517
518    /// arity trims the name. R's `read.dcf` does **not**: `Package : p`
519    /// declares a field literally named `"Package "`, so R sees no `Package`
520    /// at all. Trimming is the lenient direction, and the whitespace survives
521    /// as its own token, so a lint can still flag the typo precisely.
522    #[test]
523    fn field_name_excludes_trailing_space() {
524        let text = "Package : p\n";
525        let field = doc(text).document().fields().next().expect("a field");
526        assert_eq!(field.name(), "Package");
527        assert_eq!(field.folded_value(), "p");
528    }
529
530    /// `folded_value` would *hide* a stray `\r` (`str::trim` eats it), so CRLF
531    /// correctness has to be asserted on the tree.
532    #[test]
533    fn crlf_newline_is_one_token() {
534        let output = doc("Package: p\r\n  cont\r\n");
535        let newlines: Vec<String> = output
536            .cst
537            .descendants_with_tokens()
538            .filter_map(|el| el.into_token())
539            .filter(|t| t.kind() == SyntaxKind::NEWLINE)
540            .map(|t| t.text().to_string())
541            .collect();
542        assert_eq!(newlines, vec!["\r\n", "\r\n"]);
543
544        for token in output
545            .cst
546            .descendants_with_tokens()
547            .filter_map(|el| el.into_token())
548            .filter(|t| t.kind() == SyntaxKind::VALUE_TEXT)
549        {
550            assert!(!token.text().contains('\r'), "\\r leaked into VALUE_TEXT");
551        }
552    }
553
554    #[test]
555    fn value_keeps_colons_after_the_first() {
556        assert_eq!(
557            folded("Built: R 4.5.3; ; 2025-01-01 00:00:00 UTC; unix\n", "Built").as_deref(),
558            Some("R 4.5.3; ; 2025-01-01 00:00:00 UTC; unix")
559        );
560        assert_eq!(
561            folded(
562                "Date/Publication: 2025-09-12 07:20:14 UTC\n",
563                "Date/Publication"
564            )
565            .as_deref(),
566            Some("2025-09-12 07:20:14 UTC")
567        );
568    }
569
570    /// A field with no value at all emits no value line, and folds to `""`.
571    #[test]
572    fn field_without_a_value() {
573        let field = doc("Package:").document().fields().next().expect("a field");
574        assert_eq!(field.value_lines().count(), 0);
575        assert_eq!(field.folded_value(), "");
576    }
577
578    #[test]
579    fn raw_value_text_keeps_every_byte_after_the_colon() {
580        let field = doc("Collate:\n    a.R\n    b.R\n")
581            .document()
582            .fields()
583            .next()
584            .expect("a field");
585        assert_eq!(field.raw_value_text(), "\n    a.R\n    b.R\n");
586    }
587
588    #[test]
589    fn clean_input_reports_no_diagnostics() {
590        for input in [
591            "Package: p\n",
592            "Package: p\n\nVersion: 1\n",
593            "Collate:\n a.R\n# c\n b.R\n",
594            "   \n",
595            "# only a comment\n",
596        ] {
597            assert!(
598                doc(input).diagnostics.is_empty(),
599                "unexpected diagnostics for {input:?}: {:?}",
600                doc(input).diagnostics
601            );
602        }
603    }
604}
arity_parser/dcf/parser.rs

arity_parser/dcf/
parser.rs
