asciidork_parser/
parser.rs

1use std::fmt::Debug;
2use std::{cell::RefCell, rc::Rc};
3
4use crate::internal::*;
5
6pub struct Parser<'arena> {
7  pub(super) bump: &'arena Bump,
8  pub(super) lexer: Lexer<'arena>,
9  pub(super) document: Document<'arena>,
10  pub(super) peeked_lines: Option<ContiguousLines<'arena>>,
11  pub(super) peeked_meta: Option<ChunkMeta<'arena>>,
12  pub(super) ctx: ParseContext<'arena>,
13  pub(super) errors: RefCell<Vec<Diagnostic>>,
14  pub(super) strict: bool, // todo: naming...
15  pub(super) include_resolver: Option<Box<dyn IncludeResolver>>,
16  #[cfg(feature = "attr_ref_observation")]
17  pub(super) attr_ref_observer: Option<Box<dyn AttrRefObserver>>,
18}
19
20impl<'arena> Parser<'arena> {
21  pub fn new(src: BumpVec<'arena, u8>, file: SourceFile, bump: &'arena Bump) -> Self {
22    Parser::from_lexer(Lexer::new(src, file, bump))
23  }
24
25  pub fn from_str(src: &str, file: SourceFile, bump: &'arena Bump) -> Self {
26    Parser::from_lexer(Lexer::from_str(bump, file, src))
27  }
28
29  fn from_lexer(lexer: Lexer<'arena>) -> Self {
30    let mut parser = Parser {
31      bump: lexer.bump,
32      document: Document::new(lexer.bump),
33      peeked_lines: None,
34      peeked_meta: None,
35      ctx: ParseContext::new(lexer.bump),
36      errors: RefCell::new(Vec::new()),
37      strict: true,
38      include_resolver: None,
39      lexer,
40      #[cfg(feature = "attr_ref_observation")]
41      attr_ref_observer: None,
42    };
43    parser.set_source_file_attrs();
44    parser
45  }
46
47  pub fn apply_job_settings(&mut self, settings: JobSettings) {
48    if let Some(leveloffset) = settings.job_attrs.get("leveloffset") {
49      Parser::adjust_leveloffset(&mut self.ctx.leveloffset, &leveloffset.value);
50    }
51    self.strict = settings.strict;
52    self.ctx.max_include_depth = settings.job_attrs.u16("max-include-depth").unwrap_or(64);
53    self.document.meta = settings.into();
54    self.set_source_file_attrs();
55  }
56
57  pub fn register_plugin_macros(&mut self, names: &[impl AsRef<str>]) {
58    self.lexer.register_plugin_macros(names);
59  }
60
61  pub fn provide_timestamps(
62    &mut self,
63    now: u64,
64    input_modified_time: Option<u64>,
65    reproducible_override: Option<u64>,
66  ) {
67    self.set_datetime_attrs(now, input_modified_time, reproducible_override);
68  }
69
70  pub fn set_resolver(&mut self, resolver: Box<dyn IncludeResolver>) {
71    self.include_resolver = Some(resolver);
72  }
73
74  #[cfg(feature = "attr_ref_observation")]
75  pub fn set_attr_ref_observer(&mut self, observer: Box<dyn AttrRefObserver>) {
76    self.attr_ref_observer = Some(observer);
77  }
78
79  pub fn cell_parser(&mut self, src: BumpVec<'arena, u8>, offset: u32) -> Parser<'arena> {
80    let mut cell_parser = Parser::new(src, self.lexer.source_file().clone(), self.bump);
81    cell_parser.include_resolver = self.include_resolver.as_ref().map(|r| r.clone_box());
82    cell_parser.strict = self.strict;
83    cell_parser.lexer.adjust_offset(offset);
84    cell_parser.ctx = self.ctx.clone_for_cell(self.bump);
85    cell_parser.document.meta = self.document.meta.clone_for_cell();
86    cell_parser.document.anchors = Rc::clone(&self.document.anchors);
87
88    #[cfg(feature = "attr_ref_observation")]
89    {
90      cell_parser.attr_ref_observer = self.attr_ref_observer.take();
91    }
92
93    cell_parser
94  }
95
96  pub(crate) fn loc(&self) -> SourceLocation {
97    self
98      .peeked_lines
99      .as_ref()
100      .and_then(|lines| lines.first_loc())
101      .unwrap_or_else(|| self.lexer.loc())
102  }
103
104  pub(crate) fn read_line(&mut self) -> Result<Option<Line<'arena>>> {
105    Ok(self._read_line(false)?.map(|(line, _)| line))
106  }
107
108  fn _read_line(&mut self, ignored_last: bool) -> Result<Option<(Line<'arena>, bool)>> {
109    assert!(self.peeked_lines.is_none());
110    if self.lexer.is_eof() {
111      return Ok(None);
112    }
113
114    use TokenKind::*;
115    let mut drop_line = false;
116    let mut line = Line::empty(self.bump);
117    while !self.lexer.at_newline() && !self.lexer.is_eof() {
118      let mut token = self.lexer.next_token();
119      if line.is_empty() {
120        // if we encounter a line like `|===` in the very first paragraph,
121        // we know we're not in the header anymore, so any attrs refs can be set properly
122        if self.ctx.in_header
123          && !matches!(
124            token.kind,
125            Colon | EqualSigns | Word | ForwardSlashes | Directive | OpenBracket
126          )
127        {
128          self.ctx.in_header = false;
129        } else if token.kind == Colon && self.ctx.subs.attr_refs() {
130          self.try_parse_attr_def(&mut token)?;
131        }
132      }
133      self.push_token_replacing_attr_ref(token, &mut line, &mut drop_line)?;
134    }
135    self.lexer.skip_newline();
136    if drop_line {
137      return self._read_line(false);
138    }
139    if line.starts(TokenKind::Directive) && !self.ctx.within_block_comment() {
140      match self.try_process_directive(&mut line)? {
141        DirectiveAction::Passthrough => Ok(Some((line, ignored_last))),
142        DirectiveAction::SubstituteLine(line) => Ok(Some((line, ignored_last))),
143        DirectiveAction::IgnoreNotIncluded => self._read_line(true),
144        DirectiveAction::ReadNextLine => self._read_line(false),
145        DirectiveAction::SkipLinesUntilEndIf => Ok(
146          self
147            .skip_lines_until_endif(&line)?
148            .map(|l| (l, ignored_last)),
149        ),
150      }
151    } else {
152      Ok(Some((line, ignored_last)))
153    }
154  }
155
156  pub(crate) fn read_lines(&mut self) -> Result<Option<ContiguousLines<'arena>>> {
157    self.ctx.comment_delim_in_lines = false;
158    if let Some(peeked) = self.peeked_lines.take() {
159      return Ok(Some(peeked));
160    }
161    self.lexer.consume_empty_lines();
162    if self.lexer.is_eof() {
163      return Ok(None);
164    }
165    let mut lines = Deq::new(self.bump);
166    while let Some((line, ignored_removed_include_line)) = self._read_line(false)? {
167      if line.is_emptyish() {
168        if lines.is_empty() {
169          // this case can happen if our first non-empty line was an include directive
170          // that then resolved to an initial empty line, otherwise consume_empty_lines
171          // would have skipped over it, so we keep going
172          continue;
173        } else if !ignored_removed_include_line {
174          // this case can happen if our first non-empty line was an include directive
175          // this case happens only when we DROP a line
176          break;
177        }
178      }
179      if line.is_delimiter_kind(DelimiterKind::Comment) {
180        self.ctx.comment_delim_in_lines = true;
181      }
182      lines.push(line);
183      if self.lexer.at_newline() {
184        break;
185      }
186    }
187    if lines.is_empty() {
188      Ok(None)
189    } else {
190      Ok(Some(ContiguousLines::new(lines)))
191    }
192  }
193
194  pub(crate) fn read_lines_until(
195    &mut self,
196    delimiter: Delimiter,
197  ) -> Result<Option<ContiguousLines<'arena>>> {
198    let Some(mut lines) = self.read_lines()? else {
199      return Ok(None);
200    };
201    if lines.any(|l| l.is_delimiter(delimiter)) {
202      return Ok(Some(lines));
203    }
204
205    let mut additional_lines = BumpVec::new_in(self.bump);
206    while !self.lexer.is_eof() && !self.at_delimiter(delimiter) {
207      additional_lines.push(self.read_line()?.unwrap());
208    }
209    lines.extend(additional_lines);
210
211    while lines.last().map(|l| l.is_empty()) == Some(true) {
212      lines.pop();
213    }
214    Ok(Some(lines))
215  }
216
217  fn at_delimiter(&self, delimiter: Delimiter) -> bool {
218    match delimiter.kind {
219      DelimiterKind::BlockQuote => self.lexer.at_delimiter_line() == Some((4, b'_')),
220      DelimiterKind::Example => {
221        self.lexer.at_delimiter_line() == Some((delimiter.len as u32, b'='))
222      }
223      DelimiterKind::Open => self.lexer.at_delimiter_line() == Some((2, b'-')),
224      DelimiterKind::Sidebar => self.lexer.at_delimiter_line() == Some((4, b'*')),
225      DelimiterKind::Listing => self.lexer.at_delimiter_line() == Some((4, b'-')),
226      DelimiterKind::Literal => self.lexer.at_delimiter_line() == Some((4, b'.')),
227      DelimiterKind::Passthrough => self.lexer.at_delimiter_line() == Some((4, b'+')),
228      DelimiterKind::Comment => self.lexer.at_delimiter_line() == Some((4, b'/')),
229    }
230  }
231
232  pub(crate) fn restore_lines(&mut self, lines: ContiguousLines<'arena>) {
233    debug_assert!(self.peeked_lines.is_none());
234    if !lines.is_empty() {
235      self.peeked_lines = Some(lines);
236    }
237  }
238
239  pub(crate) fn restore_peeked_meta(&mut self, meta: ChunkMeta<'arena>) {
240    if !meta.is_empty() {
241      debug_assert!(self.peeked_meta.is_none());
242      self.peeked_meta = Some(meta);
243    }
244  }
245
246  pub(crate) fn restore_peeked(&mut self, lines: ContiguousLines<'arena>, meta: ChunkMeta<'arena>) {
247    self.restore_lines(lines);
248    self.restore_peeked_meta(meta);
249  }
250
251  pub fn parse(mut self) -> std::result::Result<ParseResult<'arena>, Vec<Diagnostic>> {
252    self.parse_document_header()?;
253    self.prepare_toc();
254
255    // ensure we only read a single "paragraph" for `inline` doc_type
256    // https://docs.asciidoctor.org/asciidoc/latest/document/doctype/#inline-doctype-rules
257    if self.document.meta.get_doctype() == DocType::Inline {
258      if self.peeked_lines.is_none() {
259        // tmp:
260        self.peeked_lines = self.read_lines().expect("tmp");
261      }
262      self.lexer.truncate();
263    }
264
265    if let Some(book_content) = self.parse_book()? {
266      self.document.content = book_content;
267    } else {
268      let sectioned = self.parse_sectioned()?;
269      self.document.content = sectioned.into_doc_content(self.bump);
270    }
271
272    // so the backend can see them replayed in decl order
273    self.document.meta.clear_doc_attrs();
274    self.diagnose_document()?;
275    Ok(self.into())
276  }
277
278  pub(crate) fn parse_sectioned(&mut self) -> Result<Sectioned<'arena>> {
279    let mut blocks = bvec![in self.bump];
280    while let Some(block) = self.parse_block()? {
281      blocks.push(block);
282    }
283    let preamble = if blocks.is_empty() { None } else { Some(blocks) };
284    let mut sections = bvec![in self.bump];
285    while let Some(section) = self.parse_section()? {
286      sections.push(section);
287    }
288    Ok(Sectioned { preamble, sections })
289  }
290
291  pub(crate) fn parse_chunk_meta(
292    &mut self,
293    lines: &mut ContiguousLines<'arena>,
294  ) -> Result<ChunkMeta<'arena>> {
295    if let Some(meta) = self.peeked_meta.take() {
296      return Ok(meta);
297    }
298    assert!(!lines.is_empty());
299    let start_loc = lines.current_token().unwrap().loc;
300    let mut attrs = MultiAttrList::new_in(self.bump);
301    let mut title = None;
302    if !lines.current().unwrap().is_fully_unconsumed() {
303      return Ok(ChunkMeta::new(attrs, title, start_loc));
304    }
305    loop {
306      match lines.current() {
307        Some(line) if line.is_chunk_title() => {
308          let mut line = lines.consume_current().unwrap();
309          line.discard_assert(TokenKind::Dots);
310          title = Some(self.parse_inlines(&mut line.into_lines())?);
311        }
312        Some(line) if line.is_block_attr_list() => {
313          let mut line = lines.consume_current().unwrap();
314          line.discard_assert(TokenKind::OpenBracket);
315          attrs.push(self.parse_block_attr_list(&mut line)?);
316        }
317        Some(line) if line.is_block_anchor() => {
318          let mut line = lines.consume_current().unwrap();
319          let first = line.discard_assert(TokenKind::OpenBracket);
320          line.discard_assert(TokenKind::OpenBracket);
321          let Some(anchor) = self.parse_block_anchor(&mut line)? else {
322            self.err_line_starting("Invalid block anchor", first.loc)?;
323            return Ok(ChunkMeta::new(attrs, title, start_loc));
324          };
325          let mut anchor_attrs = AttrList::new(anchor.loc, self.bump);
326          anchor_attrs.id = Some(anchor.id);
327          anchor_attrs.positional.push(anchor.reftext);
328          attrs.push(anchor_attrs);
329        }
330        // consume trailing comment lines for valid meta
331        Some(line) if line.is_comment() && (!attrs.is_empty() || title.is_some()) => {
332          lines.consume_current();
333        }
334        _ => break,
335      }
336    }
337    Ok(ChunkMeta::new(attrs, title, start_loc))
338  }
339
340  pub(crate) fn string(&self, s: &str) -> BumpString<'arena> {
341    BumpString::from_str_in(s, self.bump)
342  }
343}
344
345pub trait HasArena<'arena> {
346  fn bump(&self) -> &'arena Bump;
347  fn token(&self, kind: TokenKind, lexeme: &str, loc: SourceLocation) -> Token<'arena> {
348    Token::new(kind, loc, BumpString::from_str_in(lexeme, self.bump()))
349  }
350}
351
352impl<'arena> HasArena<'arena> for Parser<'arena> {
353  fn bump(&self) -> &'arena Bump {
354    self.bump
355  }
356}
357
358pub enum DirectiveAction<'arena> {
359  Passthrough,
360  ReadNextLine,
361  IgnoreNotIncluded,
362  SkipLinesUntilEndIf,
363  SubstituteLine(Line<'arena>),
364}
365
366#[derive(Debug, Clone, PartialEq, Eq)]
367pub enum SourceFile {
368  Stdin { cwd: Path },
369  Path(Path),
370  Tmp,
371}
372
373impl SourceFile {
374  pub fn file_name(&self) -> &str {
375    match self {
376      SourceFile::Stdin { .. } => "<stdin>",
377      SourceFile::Path(path) => path.file_name(),
378      SourceFile::Tmp => "<temp-buffer>",
379    }
380  }
381
382  pub fn matches_xref_target(&self, target: &str) -> bool {
383    let SourceFile::Path(path) = self else {
384      return false;
385    };
386    let filename = path.file_name();
387    if filename == target {
388      return true;
389    }
390    let xref_ext = file::ext(target);
391    let path_ext = file::ext(filename);
392    if xref_ext.is_some() && xref_ext != path_ext {
393      return false;
394    }
395    let fullpath = path.to_string();
396    if fullpath.ends_with(target) {
397      true
398    } else if xref_ext.is_some() {
399      false
400    } else {
401      file::remove_ext(&fullpath).ends_with(target)
402    }
403  }
404}
405
406impl From<Diagnostic> for Vec<Diagnostic> {
407  fn from(diagnostic: Diagnostic) -> Self {
408    vec![diagnostic]
409  }
410}
411
412#[cfg(test)]
413mod tests {
414  use super::*;
415  use test_utils::*;
416
417  fn resolve(src: &'static str) -> Box<dyn IncludeResolver> {
418    #[derive(Clone)]
419    struct MockResolver(pub Vec<u8>);
420    impl IncludeResolver for MockResolver {
421      fn resolve(
422        &mut self,
423        _: IncludeTarget,
424        buffer: &mut dyn IncludeBuffer,
425      ) -> std::result::Result<usize, ResolveError> {
426        buffer.initialize(self.0.len());
427        let bytes = buffer.as_bytes_mut();
428        bytes.copy_from_slice(&self.0);
429        Ok(self.0.len())
430      }
431      fn get_base_dir(&self) -> Option<String> {
432        Some("/".to_string())
433      }
434      fn clone_box(&self) -> Box<dyn IncludeResolver> {
435        Box::new(self.clone())
436      }
437    }
438    Box::new(MockResolver(Vec::from(src.as_bytes())))
439  }
440
441  fn reassemble(lines: ContiguousLines) -> String {
442    lines
443      .iter()
444      .map(|l| l.reassemble_src())
445      .collect::<Vec<_>>()
446      .join("\n")
447  }
448
449  #[test]
450  fn test_attr_ref() {
451    let mut parser = test_parser!("hello {foo} world");
452    parser
453      .document
454      .meta
455      .insert_doc_attr("foo", "_bar_")
456      .unwrap();
457    let mut lines = parser.read_lines().unwrap().unwrap();
458    let line = lines.consume_current().unwrap();
459    let tokens = line.into_iter().collect::<Vec<_>>();
460    expect_eq!(
461      &tokens,
462      &[
463        Token::new(TokenKind::Word, loc!(0..5), bstr!("hello")),
464        Token::new(TokenKind::Whitespace, loc!(5..6), bstr!(" ")),
465        Token::new(TokenKind::AttrRef, loc!(6..11), bstr!("{foo}")),
466        // these are inserted as an inline preprocessing step
467        // NB: we will use the source loc of the attr ref token to know how
468        // to skip over the resolve attribute in no-attr-ref subs contexts
469        Token::new(TokenKind::Underscore, loc!(6..11), bstr!("_")),
470        Token::new(TokenKind::Word, loc!(6..11), bstr!("bar")),
471        Token::new(TokenKind::Underscore, loc!(6..11), bstr!("_")),
472        // end inserted.
473        Token::new(TokenKind::Whitespace, loc!(11..12), bstr!(" ")),
474        Token::new(TokenKind::Word, loc!(12..17), bstr!("world")),
475      ]
476    );
477  }
478
479  #[test]
480  fn invalid_directive_line_passed_thru() {
481    let input = adoc! {"
482      foo
483      include::invalid []
484      bar
485    "};
486
487    let mut parser = test_parser!(input);
488    assert_eq!(
489      reassemble(parser.read_lines().unwrap().unwrap()),
490      input.trim_end()
491    );
492  }
493
494  #[test]
495  fn safe_mode_include_to_link() {
496    let input = adoc! {"
497      foo
498      include::include-file.adoc[]
499      baz
500    "};
501
502    let mut parser = test_parser!(input);
503    parser.apply_job_settings(JobSettings::secure());
504    assert_eq!(
505      reassemble(parser.read_lines().unwrap().unwrap()),
506      adoc! {"
507        foo
508        link:include-file.adoc[role=include,]
509        baz"
510      }
511    );
512
513    // assert on the tokens and positions
514    let mut parser = test_parser!(input);
515    parser.apply_job_settings(JobSettings::secure());
516
517    let mut line = parser.read_line().unwrap().unwrap();
518    expect_eq!(
519      line.consume_current().unwrap(),
520      Token::new(TokenKind::Word, loc!(0..3), bstr!("foo"))
521    );
522    assert!(line.consume_current().is_none());
523
524    assert_eq!(&input[8..13], "ude::");
525    assert_eq!(&input[30..32], "[]");
526
527    let mut line = parser.read_line().unwrap().unwrap();
528    expect_eq!(
529      std::array::from_fn(|_| line.consume_current().unwrap()),
530      [
531        // we "drop" positions 4-7, the `inc` of `include::`
532        // which becomes `••••link:`, keeping rest of token positions
533        Token::new(TokenKind::MacroName, loc!(8..13), bstr!("link:")),
534        Token::new(TokenKind::Word, loc!(13..20), bstr!("include")),
535        Token::new(TokenKind::Dashes, loc!(20..21), bstr!("-")),
536        Token::new(TokenKind::Word, loc!(21..25), bstr!("file")),
537        Token::new(TokenKind::Dots, loc!(25..26), bstr!(".")),
538        Token::new(TokenKind::Word, loc!(26..30), bstr!("adoc")),
539        Token::new(TokenKind::OpenBracket, loc!(30..31), bstr!("[")),
540        // these tokens are inserted, they have no true source so we
541        // represent their position as empty at the insertion point
542        Token::new(TokenKind::Word, loc!(31..31), bstr!("role")),
543        Token::new(TokenKind::EqualSigns, loc!(31..31), bstr!("=")),
544        Token::new(TokenKind::Word, loc!(31..31), bstr!("include")),
545        Token::new(TokenKind::Comma, loc!(31..31), bstr!(",")),
546        // /end `role=include` inserted tokens
547        Token::new(TokenKind::CloseBracket, loc!(31..32), bstr!("]")),
548      ]
549    );
550    assert!(line.consume_current().is_none());
551  }
552
553  #[test]
554  fn attrs_preserved_when_replacing_include() {
555    let input = "include::some-file.adoc[leveloffset+=1]";
556    let mut parser = test_parser!(input);
557    parser.apply_job_settings(JobSettings::secure());
558    assert_eq!(
559      parser.read_line().unwrap().unwrap().reassemble_src(),
560      "link:some-file.adoc[role=include,leveloffset+=1]"
561    );
562  }
563
564  #[test]
565  fn spaces_in_include_file_to_pass_macro_link() {
566    let input = "include::foo bar baz.adoc[]";
567    let mut parser = test_parser!(input);
568    parser.apply_job_settings(JobSettings::secure());
569    assert_eq!(
570      parser.read_line().unwrap().unwrap().reassemble_src(),
571      "link:pass:c[foo bar baz.adoc][role=include,]"
572    );
573  }
574
575  #[test]
576  fn uri_read_not_allowed_include_non_strict() {
577    // non-strict mode replaced with link
578    let input = "include::https://my.com/foo bar.adoc[]";
579    let mut parser = test_parser!(input);
580    let mut settings = JobSettings::r#unsafe();
581    settings.strict = false;
582    parser.apply_job_settings(settings);
583    expect_eq!(
584      parser.read_line().unwrap().unwrap().reassemble_src(),
585      "link:pass:c[https://my.com/foo bar.adoc][role=include,]",
586      from: input
587    );
588  }
589}
asciidork_parser/parser.rs

asciidork_parser/
parser.rs
