arity_parser/parser/

roxygen.rs

//! Roxygen2 doc-comment recognition, sub-tokenization, and block structure.
//!
//! A roxygen line is a comment whose text matches `^#+'` (one-or-more `#`
//! followed by a single `'`). Such lines are sub-tokenized—rather than emitted
//! as one `COMMENT` token—so their structure (marker, tags, arguments, prose)
//! lives directly in the lossless CST. The sub-tokens' texts tile the line's
//! bytes exactly, preserving the round-trip invariant.
//!
//! The work is split into three phases, one module each, plus this parent which
//! owns the macro-classification layer (the `\macro` arity/verbatim tables that
//! both the lexer and the structure builder consult) and the shared
//! balanced-delimiter scan:
//!
//! * [`lex`] — sub-lexing: block-mode resolution + the per-line tokenizer
//!   (text → `Vec<Token>`).
//! * [`group`] — block grouping: wrapping a run of lines in a `ROXYGEN_BLOCK`
//!   and laying out its section/paragraph skeleton (`Vec<Token>` → `Vec<Event>`).
//! * [`build`] — structure building: the block-level Rd-macro and markdown
//!   constructs (`\itemize{…}`, `\describe{…}`, markdown lists) dispatched from
//!   the grouper.

mod build;
mod group;
mod inline;
mod lex;

pub(crate) use group::emit_roxygen_block;
pub use lex::{is_md_standalone_html_tag, md_fence_run_closes, starts_md_html_block};
pub(crate) use lex::{
    is_roxygen_comment, lex_roxygen_line, rd_backslash_is_escaped, resolve_roxygen_block,
    roxygen_line_tag, scan_rd_macro, tag_body_skips_markdown, tag_folds_prose_continuation,
};

/// **Block-level** Rd macros: those whose `{…}` content R renders as its own
/// display construct — a list, a table, a preformatted or display-math block, an
/// examples-only wrapper around whole lines of R, or a conditional/sectioning
/// wrapper around any of those. Everything else (`\code`, `\emph`, `\link`,
/// `\href`, `\eqn`, the system user macros, …) is **inline** markup that belongs
/// in the surrounding prose run.
///
/// This is what makes the block/inline split a property of the *macro* rather
/// than of the author's line breaks. An unbalanced `\name{` is the only shape
/// that can span `#'` lines, so without this table "spans lines" was standing in
/// for "is a block" — which made a merely soft-wrapped `\code{…}` parse as a
/// section-level block when its opener happened to land at a line start, and as
/// inline prose markup when it did not. Both the grouper's line-start gate
/// ([`build::is_block_macro_line`]) and the formatter's layout consult this
/// instead, so the two written forms produce the same tree and the same output
/// (Tenet 1).
///
/// Names are `parse_Rd`'s, restricted to [`KNOWN_RD_MACROS`]; a package or user
/// macro is never block-level (`parse_Rd` expands it into whatever it defines).
const BLOCK_RD_MACROS: &[&str] = &[
    // Lists and tables. `\item` is a list *entry*, one per line by construction.
    "describe",
    "itemize",
    "enumerate",
    "tabular",
    "item",
    // Display text: `\preformatted` is verbatim display, `\deqn` is display math
    // (its inline sibling `\eqn` is not), `\out` is raw output R renders one
    // `VERB` atom per source line.
    "preformatted",
    "deqn",
    "out",
    // Examples-only wrappers: their content is whole lines of R, not prose.
    "dontrun",
    "donttest",
    "dontshow",
    "testonly",
    "if",
    "ifelse",
    "section",
    "subsection",
    // Whole-line declarations.
    "newcommand",
    "renewcommand",
];

/// Whether the macro named `name` (without the leading `\`) is **block-level** —
/// a display construct that owns its own lines rather than inline markup inside a
/// prose run. See [`BLOCK_RD_MACROS`]. `pub` for the formatter, which keys a
/// macro's layout on it (semver-loose, see the crate docs).
pub fn is_block_rd_macro(name: &str) -> bool {
    BLOCK_RD_MACROS.contains(&name)
}

/// Rd macros whose body's **line breaks survive rendering**, so a soft wrap
/// inside one is content rather than layout. R's renderers were probed directly
/// (`Rd2txt` over `\name{AAA,`⏎`BBB}` against `\name{AAA, BBB}`): every other
/// macro's body renders identically either way — `parse_Rd` records one atom per
/// source line, but the atoms are rejoined with a space on the way out — while
/// `\verb` and `\preformatted` render the break as a break.
///
/// This is what lets a soft-wrapped inline macro chunk exactly like the same
/// macro written on one line, instead of carrying the author's wrap into the
/// output (Tenet 1). The block-level line-significant macros (`\preformatted`,
/// `\out`, the `\dontrun` family) are in [`BLOCK_RD_MACROS`] too and never reach
/// that path, but they are named here as well so the fact stands on its own.
const LINE_SIGNIFICANT_RD_MACROS: &[&str] = &[
    "verb",
    "preformatted",
    // Raw passthrough: the body is emitted to the target format untouched, so a
    // break in it is whatever that format makes of it, not ours to normalize.
    "out",
    "Sexpr",
    // Examples-only wrappers: the body is whole lines of R.
    "dontrun",
    "donttest",
    "dontshow",
    "testonly",
];

/// Whether the macro named `name` (without the leading `\`) renders its body's
/// line breaks, so a soft wrap inside it must be preserved rather than joined.
/// See [`LINE_SIGNIFICANT_RD_MACROS`]. `pub` for the formatter, which keys reflow
/// on it (semver-loose, see the crate docs).
pub fn is_line_significant_rd_macro(name: &str) -> bool {
    LINE_SIGNIFICANT_RD_MACROS.contains(&name)
}

/// Inline Rd macros whose `{…}` content is **verbatim** (`VERB` in
/// `tools::parse_Rd`): the body is raw text and nested `\macro` markup is *not*
/// parsed. Confirmed against `parse_Rd` (see the projector's `rd_macros` work).
/// Latexlike macros (`\code`, `\emph`, `\strong`, `\link`, …) are everything
/// else --- their content is sub-parsed, so nested macros become child nodes.
/// `\eqn`/`\deqn` (LaTeX math) and `\out` (raw output) are verbatim too:
/// `parse_Rd` yields `(\eqn (VERB "x \code{q} y"))`, the nested markup literal.
const VERBATIM_RD_MACROS: &[&str] = &[
    "url", "verb", "samp", "env", "kbd", "option", "eqn", "deqn", "out",
];

/// Whether the macro named `name` (without the leading `\`) takes verbatim
/// `{…}` content. Used both when building the CST (don't recurse into a verbatim
/// body) and when projecting it (emit `VERB`, not coalesced `TEXT`) — the
/// projector's block-form verbatim arm keys on this set, so it is `pub` like its
/// [`rd_macro_arity`] sibling (semver-loose, see the crate docs).
pub fn is_verbatim_rd_macro(name: &str) -> bool {
    VERBATIM_RD_MACROS.contains(&name)
}

/// Whether argument group `index` (0-based) of the macro named `name` takes
/// **verbatim** `{…}` content (`VERB` in `parse_Rd`: raw text, no nested markup).
/// A fully-verbatim macro (`\url`/`\verb`/…) is verbatim in its only argument;
/// `\href{url}{text}` is verbatim in its *first* argument (the URL) but latexlike
/// in its *second* (the link text, which is sub-parsed). Drives both the tree
/// builder (don't recurse into a verbatim arg) and, via the emitted `VERB` leaf,
/// the projector. Confirmed against `parse_Rd`: `\href`'s first arg is `VERB`.
pub(crate) fn is_verbatim_rd_arg(name: &str, index: usize) -> bool {
    is_verbatim_rd_macro(name) || (name == "href" && index == 0) || name == "figure"
}

/// The Rd macros whose adjacent-`{…}`-group count is **not one**, with that
/// count the way `tools::parse_Rd` counts it. Every name absent here takes
/// exactly one group, so a trailing `\code{x}{y}`'s `{y}` stays literal --- the
/// arity is per macro, and a group past the count is likewise literal
/// (`\ifelse{a}{b}{c}{d}` → `(\ifelse …)` plus a sibling `(LIST …)`). Probed
/// exhaustively against R 4.6 (every [`KNOWN_RD_MACROS`] name written
/// `\name{A}{B}{C}` in a `\details` section).
///
/// A trailing group is *optional*: `\eqn{x^2}` alone is a complete call, and the
/// lexer's consumption is present-and-adjacent-only, so the optionality costs
/// nothing. parse_Rd requires same-line adjacency --- `\eqn{a} {b}` and a
/// next-line `{b}` leave the group a literal brace `LIST`. A brace-less `\item`
/// (under `\itemize`/`\enumerate`) never reaches here: it has no `{`, so it is
/// not a macro token at all.
///
/// **Arity `0` is a complete call by name alone**: parse_Rd consumes no group at
/// all, so a written `{}`/`{x}` is a *following sibling* brace list, never an
/// argument (`\dots{}` → `(\dots)` plus `(LIST)`). See [`is_zero_arg_rd_macro`].
///
/// The trailing entries are **not** parse_Rd built-ins but R's *system user
/// macros* (`share/Rd/macros/system.Rd`, always loaded before every Rd file);
/// their arity is the largest `#N` in the definition --- zero when it has none
/// --- and parse_Rd consumes that many groups before expanding. The definitions
/// themselves live in the projector's `usermacro` module --- only the group
/// count is a parsing fact.
///
/// The arity-above-one names are also the macros whose `{…}` arguments
/// `parse_Rd` models as *list* wrappers (so a multi-atom argument projects to a
/// `(GRP …)`), as opposed to latexlike macros (`\code`, `\emph`, …) whose single
/// argument's content is inlined directly. The projector keys its GRP rule on
/// [`is_multi_arg_rd_macro`].
const RD_MACRO_ARITY: &[(&str, usize)] = &[
    ("if", 2),
    ("ifelse", 3),
    ("item", 2),
    ("tabular", 2),
    ("href", 2),
    ("figure", 2),
    ("eqn", 2),
    ("deqn", 2),
    ("enc", 2),
    ("method", 2),
    ("S3method", 2),
    ("S4method", 2),
    ("subsection", 2),
    // Zero-argument built-ins: `\cr`, `\dots`, … are complete by name alone.
    ("cr", 0),
    ("tab", 0),
    ("dots", 0),
    ("ldots", 0),
    ("R", 0),
    ("manual", 2),
    ("bibinfo", 3),
    ("sspace", 0),
    ("LaTeX", 0),
];

/// How many `{…}` argument groups the macro named `name` (without the leading
/// `\`) takes; `1` for every name outside [`RD_MACRO_ARITY`]. Drives the lexer
/// (consume that many groups into one token), the tree builder (emit them as
/// children), and the projector (each group of a multi-argument macro is a list
/// argument --- a multi-atom one becomes a `(GRP …)`).
pub fn rd_macro_arity(name: &str) -> usize {
    RD_MACRO_ARITY
        .iter()
        .find(|(n, _)| *n == name)
        .map_or(1, |(_, arity)| *arity)
}

/// Whether the macro named `name` (without the leading `\`) takes more than one
/// `{…}` argument group --- parse_Rd's *structural* macros, whose arguments are
/// list wrappers. See [`rd_macro_arity`].
pub fn is_multi_arg_rd_macro(name: &str) -> bool {
    rd_macro_arity(name) > 1
}

/// R's **system user macros** (`share/Rd/macros/system.Rd`, loaded before every
/// Rd file). They are not parse_Rd built-ins --- they are `\newcommand`
/// definitions --- so they are absent from [`KNOWN_RD_MACROS`]; but parse_Rd
/// *does* consume [`rd_macro_arity`] groups for each before expanding it, which
/// makes their names a parsing fact. Without this list the lexer could not tell
/// `\doi{10.1/x}` (one consumed argument) from a genuinely unrecognized
/// `\zzz{x}` (no argument at all --- see [`is_argument_taking_rd_macro`]).
///
/// The **definitions** live in the projector's `usermacro` module, which pins
/// this list against them; only the names and group counts are parsing facts.
const SYSTEM_RD_USER_MACROS: &[&str] = &[
    "CRANpkg",
    "PR",
    "doi",
    "I",
    "packageTitle",
    "packageDescription",
    "packageAuthor",
    "packageMaintainer",
    "packageDESCRIPTION",
    "packageIndices",
    "bibcitep",
    "bibcitet",
    "bibshow",
    "proglang",
    "sspace",
    "LaTeX",
    "manual",
    "bibinfo",
];

/// Every name in [`SYSTEM_RD_USER_MACROS`], for the projector's drift check
/// against its own definition table.
pub fn system_rd_user_macro_names() -> &'static [&'static str] {
    SYSTEM_RD_USER_MACROS
}

/// Whether `parse_Rd` consumes `{…}` argument groups for `\name` --- a built-in
/// ([`is_known_rd_macro`]) or a system user macro ([`SYSTEM_RD_USER_MACROS`]).
/// Everything else is an unrecognized `\word`: parse_Rd tags it `UNKNOWN` and
/// consumes **nothing**, so a written group is a following sibling brace list
/// (`\zzz{x}` → `(UNKNOWN "\zzz") (LIST (TEXT "x"))`), exactly like a zero-arity
/// call's. Arity still applies on top --- a zero-arity name in either set takes
/// no group either.
pub fn is_argument_taking_rd_macro(name: &str) -> bool {
    is_known_rd_macro(name) || SYSTEM_RD_USER_MACROS.contains(&name)
}

/// Split a GFM table row into its cells, honoring backslash-escaped pipes. One
/// optional leading and one optional trailing **unescaped** `|` are stripped (the
/// GFM leading/trailing pipe), then the remainder is split on each unescaped `|`.
/// Cells are returned untrimmed (callers trim). An escaped `\|` stays inside its
/// cell. Shared by the recognition gate (cell counting) and the projector (cell
/// rendering) so the two never disagree on where a cell begins.
///
/// GFM counts pipes **without** honoring code spans — a `|` inside `` `…` `` still
/// splits a cell — so this deliberately does not track backticks. That is what
/// makes `| ` + "`a|b`" + ` | y |` fail the header/delimiter cell-count match and
/// stay prose (verified against roxygen2).
pub fn split_table_row_cells(line: &str) -> Vec<&str> {
    let trimmed = line.trim();
    let bytes = trimmed.as_bytes();
    let start = usize::from(bytes.first() == Some(&b'|'));
    let end = if bytes.len() > start
        && bytes[bytes.len() - 1] == b'|'
        && !pipe_is_escaped(bytes, bytes.len() - 1)
    {
        bytes.len() - 1
    } else {
        bytes.len()
    };
    let inner = &trimmed[start..end];
    let ib = inner.as_bytes();
    let mut cells = Vec::new();
    let mut cell_start = 0;
    let mut i = 0;
    while i < ib.len() {
        match ib[i] {
            b'\\' => i += 2, // skip the escaped byte (an escaped `\|` stays in-cell)
            b'|' => {
                cells.push(&inner[cell_start..i]);
                i += 1;
                cell_start = i;
            }
            _ => i += 1,
        }
    }
    cells.push(&inner[cell_start.min(inner.len())..]);
    cells
}

/// The number of cells in a GFM table row (see [`split_table_row_cells`]). The
/// header row and the delimiter row form a table only when these are equal.
pub(crate) fn count_table_cells(line: &str) -> usize {
    split_table_row_cells(line).len()
}

/// Whether `line` is a GFM table **delimiter row**: it contains at least one
/// unescaped `|` (so a bare `---`, which is a setext underline, is *not* a
/// single-column table) and every cell (trimmed) is `:?-+:?` (optional leading
/// colon, one or more hyphens, optional trailing colon). The pipe requirement
/// mirrors cmark-gfm, which treats a pipeless dash run as a setext heading.
pub fn is_table_delim_row(line: &str) -> bool {
    let trimmed = line.trim();
    if !line_has_unescaped_pipe(trimmed) {
        return false;
    }
    let cells = split_table_row_cells(trimmed);
    !cells.is_empty() && cells.iter().all(|c| is_table_delim_cell(c.trim()))
}

/// Whether the byte at `idx` (a `|`) is backslash-escaped: preceded by an odd
/// run of `\`.
fn pipe_is_escaped(bytes: &[u8], idx: usize) -> bool {
    let mut k = idx;
    let mut count = 0;
    while k > 0 && bytes[k - 1] == b'\\' {
        count += 1;
        k -= 1;
    }
    count % 2 == 1
}

/// Whether `line` contains an unescaped `|`.
fn line_has_unescaped_pipe(line: &str) -> bool {
    let bytes = line.as_bytes();
    let mut i = 0;
    while i < bytes.len() {
        match bytes[i] {
            b'\\' => i += 2,
            b'|' => return true,
            _ => i += 1,
        }
    }
    false
}

/// Whether `cell` (already trimmed) is a valid delimiter cell: `:?-+:?`.
fn is_table_delim_cell(cell: &str) -> bool {
    let b = cell.as_bytes();
    let mut i = usize::from(b.first() == Some(&b':'));
    let dash_start = i;
    while i < b.len() && b[i] == b'-' {
        i += 1;
    }
    if i == dash_start {
        return false; // a delimiter cell needs at least one hyphen
    }
    if b.get(i) == Some(&b':') {
        i += 1;
    }
    i == b.len()
}

/// Scan a balanced delimited run starting at `bytes[i] == open`, tracking nesting
/// and skipping Rd backslash escapes (`\}` etc.). Returns the index past the
/// matching `close`, or `None` if it is unbalanced before end of input.
pub(crate) fn scan_balanced(bytes: &[u8], i: usize, open: u8, close: u8) -> Option<usize> {
    debug_assert_eq!(bytes[i], open);
    let mut depth = 0usize;
    let mut j = i;
    while j < bytes.len() {
        let b = bytes[j];
        if b == b'\\' {
            j += 2; // skip the escaped byte
        } else if b == open {
            depth += 1;
            j += 1;
        } else if b == close {
            depth -= 1;
            j += 1;
            if depth == 0 {
                return Some(j);
            }
        } else {
            j += 1;
        }
    }
    None
}

/// Advance a markdown value column across one whitespace character: a tab
/// expands to the next 4-column tab stop (CommonMark's rule — roxygen2 hands
/// cmark the tag value with tabs intact), any other character is one column.
/// Columns are zero-based **value** coordinates: column 0 is the first
/// character after the `#'` sigil plus the one whitespace character roxygen2's
/// tokenizer strips (`consumeWhitespace(1)`, parser2.cpp — a space *or* a tab).
/// The single source of column arithmetic for the block builder's indent gauge
/// and the projector's column stripping.
pub fn advance_md_col(col: usize, c: char) -> usize {
    if c == '\t' {
        col - col % 4 + 4
    } else {
        col + 1
    }
}

/// The one-based indent **gauge** of a marker→content whitespace run: the first
/// character (space or tab alike) is the one roxygen2 strips and maps to
/// gauge 1 (value column 0); the rest expand via [`advance_md_col`], so the
/// gauge is the content's value column plus one. An empty run gauges 0. For an
/// all-space run this equals the character count — the pre-tab behavior.
pub fn md_ws_gauge<'a>(texts: impl IntoIterator<Item = &'a str>) -> usize {
    let mut col = 0usize;
    let mut first = true;
    for text in texts {
        for c in text.chars() {
            if first {
                first = false;
                continue;
            }
            col = advance_md_col(col, c);
        }
    }
    if first { 0 } else { col + 1 }
}

/// The end index of an Rd macro name starting at `bytes[start]` (the byte *after*
/// the leading `\`). An Rd command name is `[A-Za-z][A-Za-z0-9]*`: a leading
/// letter then any letters or digits (e.g. `\linkS4class`). Returns `start` when
/// no valid name begins there (`\\`, `\{`, `\4`, end of input). The single source
/// of truth for where a `\name` ends, shared by the lexer and the tree builder.
pub fn rd_macro_name_end(bytes: &[u8], start: usize) -> usize {
    let mut k = start;
    if bytes.get(k).is_some_and(u8::is_ascii_alphabetic) {
        k += 1;
        while k < bytes.len() && bytes[k].is_ascii_alphanumeric() {
            k += 1;
        }
    }
    k
}

/// The built-in Rd macro names `tools::parse_Rd` recognizes (without the leading
/// `\`). A `\word` *not* in this set is an **unknown** macro: `parse_Rd` tags it
/// `UNKNOWN` (warning "unknown macro '\word'"), even brace-less. Used to gate
/// brace-less macro recognition in the lexer (only an unknown name is carved as a
/// token; a known name brace-less stays literal prose --- its name-only/expanded
/// rendering is backlog) and the projector's name-only classification (a known
/// list child like `\item`/`\cr` → `(\name)`, an unknown one → `(UNKNOWN …)`).
///
/// The set is parse_Rd's static keyword table, verified against R 4.5; it
/// deliberately excludes package/user-defined macros (`\CRANpkg`, `\doi`, …),
/// which `parse_Rd` *expands* rather than parses (out of scope for a static
/// projector --- they surface as faithful divergences).
const KNOWN_RD_MACROS: &[&str] = &[
    // Sectioning / structural commands.
    "name",
    "alias",
    "title",
    "description",
    "usage",
    "arguments",
    "value",
    "details",
    "references",
    "note",
    "author",
    "seealso",
    "examples",
    "keyword",
    "concept",
    "section",
    "subsection",
    "docType",
    "encoding",
    "Rdversion",
    "format",
    "source",
    "synopsis",
    "figure",
    "item",
    "describe",
    "itemize",
    "enumerate",
    "tabular",
    "method",
    "S3method",
    "S4method",
    "newcommand",
    "renewcommand",
    "Sexpr",
    "RdOpts",
    "if",
    "ifelse",
    "out",
    "enc",
    "href",
    // Inline text / cross-reference / math macros.
    "emph",
    "strong",
    "bold",
    "code",
    "preformatted",
    "kbd",
    "samp",
    "pkg",
    "file",
    "email",
    "url",
    "var",
    "env",
    "option",
    "command",
    "dfn",
    "cite",
    "acronym",
    "dQuote",
    "sQuote",
    "verb",
    "link",
    "linkS4class",
    "eqn",
    "deqn",
    // Zero-argument / escape / examples-only commands.
    "cr",
    "tab",
    "dots",
    "ldots",
    "R",
    "dontrun",
    "donttest",
    "dontshow",
    "testonly",
];

/// Whether `name` (without the leading `\`) is a built-in Rd macro `parse_Rd`
/// recognizes. The single source of truth for the known/unknown split, shared by
/// the lexer (gate brace-less recognition) and the projector (name-only → `(\name)`
/// vs `(UNKNOWN …)`). See [`KNOWN_RD_MACROS`].
pub fn is_known_rd_macro(name: &str) -> bool {
    KNOWN_RD_MACROS.contains(&name)
}

/// Whether the macro named `name` (without the leading `\`) takes **no**
/// argument group --- `parse_Rd`'s zero-argument built-ins (`\dots`, `\cr`, …)
/// and the zero-placeholder system user macros (`\sspace`, `\LaTeX`). Such an
/// occurrence is a *complete* call by its name alone, so the lexer carves it
/// name-only even when a `{` follows (the group is then a sibling brace list) —
/// unlike other known names, whose brace-less misuse triggers parse_Rd's
/// drop-recovery. One source of truth with the rest of the group count:
/// [`RD_MACRO_ARITY`].
pub(crate) fn is_zero_arg_rd_macro(name: &str) -> bool {
    rd_macro_arity(name) == 0
}

/// Built-in Rd macros whose brace-less misuse does **not** recover as a clean
/// text drop: by the time parse_Rd's "expecting `{`" error fires, its lexer has
/// already switched to the macro's argument mode, and the recovery leaves that
/// state in place — everything to *section end* (crossing paragraph breaks)
/// becomes per-line `RCODE`/`VERB` atoms (`\code z` → `(RCODE " z after\n") …`).
/// `\item` is special the other way: out of list context parse_Rd tags it an
/// `(UNKNOWN "\item")` node and the text continues. Probed exhaustively against
/// R 4.5 (every [`KNOWN_RD_MACROS`] name as `before \name z after`); every name
/// *not* here (and not zero-arg) drops cleanly instead — see
/// [`is_rd_braceless_drop_macro`].
///
/// This list covers the **built-ins** only; the argument-taking *system user
/// macros* are sticky too, and are folded in by
/// [`is_sticky_braceless_rd_macro`] rather than repeated here.
const STICKY_BRACELESS_RD_MACROS: &[&str] = &[
    // RCODE-argument macros: recovery leaves parse_Rd in R-code mode.
    "code",
    "donttest",
    "dontshow",
    "testonly",
    // VERB-argument macros: recovery leaves parse_Rd in verbatim mode.
    "verb",
    "url",
    "samp",
    "env",
    "kbd",
    "option",
    "out",
    "eqn",
    "deqn",
    "href",
    "figure",
    "preformatted",
    "dontrun",
    "newcommand",
    "renewcommand",
    // Out-of-list `\item` becomes an `(UNKNOWN …)` node, not a drop.
    "item",
];

/// Whether the brace-less misuse of `\name` leaves parse_Rd in its argument
/// mode, swallowing the rest of the section — the built-ins of
/// [`STICKY_BRACELESS_RD_MACROS`] plus every **argument-taking system user
/// macro** ([`SYSTEM_RD_USER_MACROS`]).
///
/// A system user macro is sticky for the same reason a built-in is: parse_Rd
/// consumes its groups ([`is_argument_taking_rd_macro`]) *before* expanding it,
/// so the "expecting `{`" recovery fires with the lexer already switched — and
/// it is always the **verbatim** mode, never R-code (probed against R 4.6:
/// every name in the list as `before \name b after`). The zero-arity ones
/// (`\sspace`, `\LaTeX`) are excluded: they need no group, so they simply expand
/// on their name and the prose continues.
pub(crate) fn is_sticky_braceless_rd_macro(name: &str) -> bool {
    STICKY_BRACELESS_RD_MACROS.contains(&name)
        || (SYSTEM_RD_USER_MACROS.contains(&name) && !is_zero_arg_rd_macro(name))
}

/// Whether a brace-less `\name` in prose is dropped by parse_Rd's recovery with
/// the surrounding text continuing as plain `TEXT` (the "unexpected TEXT/section
/// header, expecting `{`" recovery): an argument-taking, non-zero-arg macro
/// outside [`is_sticky_braceless_rd_macro`]. Zero-arg names are complete calls
/// (carved by the lexer), unknown names are `(UNKNOWN …)` nodes (also carved),
/// and the sticky set leaves parse_Rd in a code/verbatim mode. Of that set,
/// brace-less `\item` is projected as an `(UNKNOWN "\item")` node
/// (`split_braceless_items` in the projector) and the code/verbatim mode-flip
/// names swallow the section (`split_sticky_braceless_swallow`).
/// Mode-independent: the `@md` pipeline is a net no-op on a backslash run before
/// a letter, so parse_Rd sees the same brace-less macro either way.
pub fn is_rd_braceless_drop_macro(name: &str) -> bool {
    is_argument_taking_rd_macro(name)
        && !is_zero_arg_rd_macro(name)
        && !is_sticky_braceless_rd_macro(name)
}

/// For a brace-less sticky Rd macro name (`\code`/`\verb`/`\doi`/…, see
/// [`is_sticky_braceless_rd_macro`]), which parse_Rd lexer state its "expecting
/// `{`" recovery leaves in place: `Some(true)` for the **R-code** macros (the
/// recovery stays in R-code mode → per-line `RCODE` atoms) and `Some(false)` for
/// the **verbatim** macros (verbatim mode → per-line `VERB` atoms). `None` for a
/// non-sticky name and for `item`, whose out-of-list misuse is instead an
/// `(UNKNOWN "\item")` node (handled by the projector's `split_braceless_items`).
/// The two built-in sets are exactly [`STICKY_BRACELESS_RD_MACROS`] minus `item`,
/// probed against R 4.5 (see that list's doc); the argument-taking system user
/// macros are uniformly verbatim, probed against R 4.6.
pub fn sticky_braceless_code_mode(name: &str) -> Option<bool> {
    match name {
        "code" | "donttest" | "dontshow" | "testonly" => Some(true),
        "verb" | "url" | "samp" | "env" | "kbd" | "option" | "out" | "eqn" | "deqn" | "href"
        | "figure" | "preformatted" | "dontrun" | "newcommand" | "renewcommand" => Some(false),
        // `\item` is the one sticky name with no argument mode to leave behind.
        "item" => None,
        _ => {
            (SYSTEM_RD_USER_MACROS.contains(&name) && !is_zero_arg_rd_macro(name)).then_some(false)
        }
    }
}

/// Rd macros whose `{…}` content roxygen2 **protects** from the markdown parser
/// (`escaped_for_md` in roxygen2's `R/markdown-escaping.R`): under `@md`,
/// `escape_rd_for_md` swaps the whole `\tag{…}` span out for a placeholder before
/// running cmark, so markdown inside such a macro stays literal Rd
/// (`\code{*x*}` → `\code{*x*}`, not `\code{\emph{x}}`). Every *other* macro keeps
/// only its backslash-word as literal text while its argument **is** markdown-
/// processed (`\emph{*x*}` → `\emph{\emph{x}}`), so the projector resolves the arg
/// of a non-fragile, known, single-argument macro as a markdown inline run.
const FRAGILE_FOR_MD_RD_MACROS: &[&str] = &[
    "acronym",
    "code",
    "command",
    "CRANpkg",
    "deqn",
    "doi",
    "dontrun",
    "dontshow",
    "donttest",
    "email",
    "env",
    "eqn",
    "figure",
    "file",
    "if",
    "ifelse",
    "kbd",
    "link",
    "linkS4class",
    "method",
    "mjeqn",
    "mjdeqn",
    "mjseqn",
    "mjsdeqn",
    "mjteqn",
    "mjtdeqn",
    "newcommand",
    "option",
    "out",
    "packageAuthor",
    "packageDescription",
    "packageDESCRIPTION",
    "packageIndices",
    "packageMaintainer",
    "packageTitle",
    "pkg",
    "PR",
    "preformatted",
    "renewcommand",
    "S3method",
    "S4method",
    "samp",
    "special",
    "testonly",
    "url",
    "var",
    "verb",
];

/// Whether the macro named `name` (without the leading `\`) has its `{…}` content
/// **protected** from markdown under `@md` (roxygen2's `escaped_for_md`). A fragile
/// macro keeps its argument literal; a non-fragile one has it markdown-processed.
/// See [`FRAGILE_FOR_MD_RD_MACROS`].
pub fn is_fragile_for_md(name: &str) -> bool {
    FRAGILE_FOR_MD_RD_MACROS.contains(&name)
}

/// Resolve a bare prose `content` string as a `@md` markdown **inline run** and
/// return the resulting `ROXYGEN_PARAGRAPH` node, whose children are the resolved
/// inline elements (text, emphasis/strong nodes, links, code spans, nested Rd
/// macros). Drives the projector's translation of a non-fragile Rd macro's argument
/// under `@md` (`\emph{*x*}` → `\emph{\emph{x}}`): the projector slices out the raw
/// argument text and feeds it here, reusing the **real** inline pass (the
/// delimiter-stack arena) rather than a second markdown scanner — so nesting,
/// links, and code spans resolve exactly as in ordinary `@md` prose. A nested
/// fragile macro stays an opaque `ROXYGEN_RD_MACRO` token here; the projector keeps
/// its argument literal by recursing with the same fragility check.
pub fn resolve_md_inline(content: &str) -> crate::syntax::SyntaxNode {
    let mut tokens = Vec::new();
    lex::lex_roxygen_prose_fragment(&mut tokens, content, true);
    resolve_md_inline_tokens(tokens, true)
}

/// Non-markdown twin of [`resolve_md_inline`]: lex `content` as a plain **Rd**
/// prose fragment (no markdown recognizers; `\name{…}` macros still carve, gated
/// by the backslash-pairing parity) and return the `ROXYGEN_PARAGRAPH` node.
/// Drives the projector's re-parse of a *demoted* markdown span's rendered body,
/// which `tools::parse_Rd` reads as ordinary Rd text once the generating macro's
/// backslash has been paired away.
pub fn resolve_rd_inline(content: &str) -> crate::syntax::SyntaxNode {
    let mut tokens = Vec::new();
    lex::lex_roxygen_prose_fragment(&mut tokens, content, false);
    resolve_md_inline_tokens(tokens, false)
}

/// One piece of a **structural** Rd-macro argument fed to
/// [`resolve_md_inline_pieces`]: either raw prose `Text` (markdown-lexed) or a
/// pre-parsed nested `Macro` (its raw `\name{…}`/`\name` source, kept opaque).
pub enum MdArgPiece {
    /// Raw prose text, lexed as a markdown inline fragment.
    Text(String),
    /// A nested Rd macro's raw source (`\strong{y}`, `\tab`, `\cr`, …), emitted as
    /// one opaque `RoxygenRdMacro` token so emphasis/links span across it.
    Macro(String),
}

/// Resolve a **structural** Rd-macro argument (`\item`/`\tabular`/`\href` under
/// `@md`) as a single markdown inline run from its already-carved `pieces`,
/// returning the resolved `ROXYGEN_PARAGRAPH` node (same shape as
/// [`resolve_md_inline`]).
///
/// roxygen2 markdown-processes a structural argument as **one** cmark run: a nested
/// Rd macro is opaque text to cmark (reconstituted afterward), so an emphasis or
/// link span crosses it. Re-lexing the raw argument string cannot reproduce this
/// faithfully — the prose fragment lexer leaves a *brace-less* known macro
/// (`\tab`/`\cr`, the table separators) literal. Instead each pre-parsed macro
/// child (carved by the block-macro grouper) is emitted as one opaque
/// `RoxygenRdMacro` token, which [`build_rd_macro`](crate::parser::tree_builder)
/// re-expands into a faithful node (a brace-less `\tab` → a name-only `\tab` node).
/// The prose pieces between them lex as ordinary markdown fragments, so the
/// delimiter-stack arena spans emphasis across the macros exactly as cmark does.
pub fn resolve_md_inline_pieces(pieces: &[MdArgPiece]) -> crate::syntax::SyntaxNode {
    use crate::parser::lexer::{TokKind, Token};

    let mut tokens = Vec::new();
    for piece in pieces {
        match piece {
            MdArgPiece::Text(t) => lex::lex_roxygen_prose_fragment(&mut tokens, t, true),
            // Offsets are unused by the inline pass and the tree builder (which key
            // off `kind`/`text` only), so a synthetic macro token needs no real span.
            MdArgPiece::Macro(m) => tokens.push(Token {
                kind: TokKind::RoxygenRdMacro,
                text: m,
                start: 0,
                end: 0,
            }),
        }
    }
    resolve_md_inline_tokens(tokens, true)
}

/// Wrap `tokens` in a paragraph, run the emphasis/inline pass, and return the
/// resolved `ROXYGEN_PARAGRAPH` node. Shared by [`resolve_md_inline`],
/// [`resolve_rd_inline`], and [`resolve_md_inline_pieces`]. `md` is the mode the
/// tokens were lexed under; a non-md fragment carries no markdown delimiters, so
/// the inline pass only threads its tokens through.
fn resolve_md_inline_tokens(
    tokens: Vec<crate::parser::lexer::Token>,
    md: bool,
) -> crate::syntax::SyntaxNode {
    use crate::parser::events::Event;
    use crate::syntax::SyntaxKind;

    let mut events = Vec::with_capacity(tokens.len() + 2);
    events.push(Event::Start(SyntaxKind::ROXYGEN_PARAGRAPH));
    events.extend((0..tokens.len()).map(Event::Tok));
    events.push(Event::Finish);
    // Fragments carry no soft breaks, so the multi-line HTML resolution is
    // inert here.
    inline::resolve_emphasis(&tokens, &mut events, md);
    let root = crate::parser::tree_builder::build_tree(&tokens, &events);
    root.first_child()
        .expect("build_tree always wraps the paragraph in ROOT")
}

/// Length in bytes of the UTF-8 char whose leading byte is `b`.
fn utf8_len(b: u8) -> usize {
    match b {
        0x00..=0x7F => 1,
        0xC0..=0xDF => 2,
        0xE0..=0xEF => 3,
        _ => 4,
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    /// The brace-less classification is a three-way split, and every
    /// argument-taking name must land in exactly one arm: sticky (parse_Rd's
    /// recovery leaves an argument mode behind), clean drop, or — for the
    /// zero-arity and unknown names — not a misuse at all.
    #[test]
    fn braceless_classification_is_a_partition() {
        for name in KNOWN_RD_MACROS.iter().chain(SYSTEM_RD_USER_MACROS) {
            if is_zero_arg_rd_macro(name) {
                assert!(
                    !is_sticky_braceless_rd_macro(name),
                    "{name} zero-arg sticky"
                );
                assert!(!is_rd_braceless_drop_macro(name), "{name} zero-arg drop");
                continue;
            }
            assert_ne!(
                is_sticky_braceless_rd_macro(name),
                is_rd_braceless_drop_macro(name),
                "{name} is both or neither sticky and drop",
            );
        }
        // An unrecognized name is an `(UNKNOWN …)` node, neither of the two.
        assert!(!is_sticky_braceless_rd_macro("zzz"));
        assert!(!is_rd_braceless_drop_macro("zzz"));
    }

    /// Every argument-taking system user macro is sticky, and uniformly in the
    /// **verbatim** mode — probed against R 4.6 (`before \name b after` for each).
    /// The zero-arity ones expand on their name, so they are not sticky at all.
    #[test]
    fn argument_taking_system_macros_are_verbatim_sticky() {
        for name in SYSTEM_RD_USER_MACROS {
            if is_zero_arg_rd_macro(name) {
                assert_eq!(sticky_braceless_code_mode(name), None, "{name}");
            } else {
                assert_eq!(sticky_braceless_code_mode(name), Some(false), "{name}");
            }
        }
        // The built-ins keep both modes, and `\item` neither.
        assert_eq!(sticky_braceless_code_mode("code"), Some(true));
        assert_eq!(sticky_braceless_code_mode("verb"), Some(false));
        assert_eq!(sticky_braceless_code_mode("item"), None);
    }

    /// `sticky_braceless_code_mode` answers for exactly the sticky names minus
    /// `item`, so the mode table and the set cannot drift apart.
    #[test]
    fn sticky_set_and_mode_table_agree() {
        for name in KNOWN_RD_MACROS.iter().chain(SYSTEM_RD_USER_MACROS) {
            let expected = is_sticky_braceless_rd_macro(name) && *name != "item";
            assert_eq!(
                sticky_braceless_code_mode(name).is_some(),
                expected,
                "{name}",
            );
        }
    }
}