prov_graph/

content.rs

//! Body-prose parsing via `twig` — prov's answer to the `content_format`
//! knob deferred in `docs/next-steps.md`, and the ingredient that makes
//! body-link findings code-aware (DESIGN §8's principle: a `[[…]]` that is
//! really code, e.g. `[[inf] * n for _ in range(m)]]` inside backticks, must
//! never be treated as a link).
//!
//! `twig` (a sister Zig-backed project) parses Markdown/Djot into a shared AST.
//! [`render_html`] and [`code_spans`] are direct FFI calls into it — `twig`'s C
//! ABI exposes `twig_document_render_html` and `twig_document_nodes`, no
//! subprocess involved. (`code_spans` used to bind a code-block-specific
//! accessor, then a selector query per code-bearing kind; it now filters the
//! flat node array by kind, which needs one call and reaches the detached
//! definition subtrees a query does not — see `spans_where`.) `twig` is a
//! required dependency, so these are always available.
//!
//! Pair [`code_spans`] with [`crate::link::scan_wikilinks`] (which is what
//! actually uses it) to keep a body-link scan from ever treating code as
//! prose.

use std::ops::Range;
use std::path::Path;

/// Which body-prose grammar a document is written in. Maps to a `twig`
/// [`twig::Format`] one-to-one; kept as prov's own type so callers can name
/// a format without depending on `twig` directly, e.g. for the `content_format`
/// config knob.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ContentFormat {
    Markdown,
    Djot,
    Html,
}

impl ContentFormat {
    /// Infer the content format from a path's extension. `None` for anything
    /// unrecognized (including config extensions, which have no body).
    pub fn from_extension(path: &Path) -> Option<Self> {
        match path.extension()?.to_str()? {
            "md" | "markdown" => Some(Self::Markdown),
            "dj" | "djot" => Some(Self::Djot),
            "html" | "htm" => Some(Self::Html),
            _ => None,
        }
    }

    fn twig_format(self) -> twig::Format {
        match self {
            Self::Markdown => twig::Format::Markdown,
            Self::Djot => twig::Format::Djot,
            Self::Html => twig::Format::Html,
        }
    }

    /// The canonical file extension for this grammar (no leading dot) — what a
    /// freshly authored document's filename gets when prov derives a name
    /// from a title (`prov new "A Title"`). The inverse of the primary
    /// [`from_extension`](Self::from_extension) spelling.
    pub fn extension(self) -> &'static str {
        match self {
            Self::Markdown => "md",
            Self::Djot => "dj",
            Self::Html => "html",
        }
    }

    /// The `content_format` config-document spelling for this grammar.
    pub fn as_config_str(self) -> &'static str {
        match self {
            Self::Markdown => "markdown",
            Self::Djot => "djot",
            Self::Html => "html",
        }
    }

    /// Parse a `content_format` config value. Unknown → `None` (keep default).
    pub fn from_config_str(value: &str) -> Option<Self> {
        match value {
            "markdown" | "md" => Some(Self::Markdown),
            "djot" | "dj" => Some(Self::Djot),
            "html" | "htm" => Some(Self::Html),
            _ => None,
        }
    }

    /// Whether [transcoding](transcode) between `self` and `other` loses authored
    /// structure badly enough to need an explicit `--force`.
    ///
    /// HTML is the lossy endpoint, in both directions. *Into* HTML is a one-way
    /// trip: the result is a rendering, and the Markdown or Djot the author wrote
    /// — the `#`, the `_emph_`, the fence — is gone from the file, recoverable
    /// only by re-deriving a guess at it. *Out of* HTML is that trip run
    /// backwards, and everything HTML carries that a prose grammar has no spelling
    /// for (attributes, nested inline markup, whole elements) survives only as a
    /// raw-HTML escape, or not at all.
    ///
    /// Markdown ↔ Djot is deliberately not gated: twig re-spells emphasis,
    /// headings and raw HTML into the target grammar and carries footnotes,
    /// tables, code fences and `[[wikilinks]]` through intact. The one wart is a
    /// reference-style link, which is inlined (`[x][ref]` → `[x](notes/b.md)`),
    /// leaving its now-unused `[ref]:` definition behind — untidy, but nothing a
    /// reader loses.
    pub fn is_lossy_to(self, other: Self) -> bool {
        self != other && (self == Self::Html || other == Self::Html)
    }
}

/// Parse `body` as `format` with `twig`. Shared by [`render_html`] and
/// [`code_spans`] so both go through the same error mapping.
fn parse(body: &str, format: ContentFormat) -> crate::error::Result<twig::Document> {
    twig::Document::parse_str(body, format.twig_format())
        .map_err(|e| crate::error::Error::Content(format!("twig parse: {e}")))
}

/// Transcode a body from the `from` grammar into the `to` grammar.
///
/// Two callers, one move. It is what every page prov *authors* rather than reads
/// goes through — `prov`'s `about` page, the history store's index and event
/// bodies are written as Markdown in the Rust source, where they are legible to
/// whoever maintains them, and converted here to whatever grammar the workspace
/// actually uses (without which an HTML workspace ends up holding `.html` files
/// whose bodies are literal `# Heading` Markdown: prov reads them back fine, and
/// every other tool in the world does not). It is also the engine behind
/// `convert <file> content_format`, where `from` is the grammar the document's
/// own extension declares rather than always Markdown.
///
/// A body already in the target grammar is returned untouched: twig's serializer
/// is idempotent here, but round-tripping would buy nothing and risks reflowing
/// prose the author deliberately wrapped.
pub fn transcode(
    body: &str,
    from: ContentFormat,
    to: ContentFormat,
) -> crate::error::Result<String> {
    if from == to {
        return Ok(body.to_string());
    }
    let mut doc = parse(body, from)?;
    let out = doc
        .serialize(to.twig_format())
        .map_err(|e| crate::error::Error::Content(format!("twig serialize: {e}")))?;
    String::from_utf8(out)
        .map_err(|e| crate::error::Error::Content(format!("twig produced non-UTF-8: {e}")))
}

/// Parse `body` as `format` and render it to HTML, via `twig`'s FFI.
pub fn render_html(body: &str, format: ContentFormat) -> crate::error::Result<String> {
    let mut doc = parse(body, format)?;
    let html = doc
        .render_html()
        .map_err(|e| crate::error::Error::Content(format!("twig render: {e}")))?;
    String::from_utf8(html)
        .map_err(|e| crate::error::Error::Content(format!("twig produced non-UTF-8 HTML: {e}")))
}

/// Whether a node kind is one `twig` parses as opaque code — inline code spans
/// (`Verbatim`), fenced/indented code blocks (`CodeBlock`), and raw
/// inline/block escapes (`RawInline` / `RawBlock`).
fn is_code(kind: &twig::Kind) -> bool {
    matches!(
        kind,
        twig::Kind::Verbatim | twig::Kind::CodeBlock | twig::Kind::RawInline | twig::Kind::RawBlock
    )
}

/// What one parse of a body yields for a link scan: the spans twig reads as
/// code, the spans it reads as inline links, and the spans it reads as images,
/// each sorted by start offset.
///
/// A named struct rather than a bare tuple because the lists are the same
/// shape, and telling them apart at a call site should not depend on getting
/// their order right.
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct BodySpans {
    /// Everything a link scan must treat as opaque — see [`code_spans`].
    pub code: Vec<Range<usize>>,
    /// Each inline `[label](target)` construct — see [`link_spans`].
    pub links: Vec<Range<usize>>,
    /// Each `![alt](target)` construct, the leading `!` included. twig parses an
    /// image as its own node kind, not as a link, so a scan that collected only
    /// [`Self::links`] never saw one — and a move rewrote every link in a page's
    /// body while leaving its pictures pointing at the old directory.
    pub images: Vec<Range<usize>>,
}

/// The code spans, the inline-link spans and the image spans of `body`, in
/// **one parse**.
///
/// All are read off the same node array, because a body-link scan wants each
/// and twig's parse is the expensive part: [`crate::link::scan_body_links`]
/// asked for them separately and so parsed every document's prose twice, which
/// a profile of `check` put at 41% of the run. The two lists come back sorted
/// by start offset, each exactly as its single-kind accessor would have
/// returned it.
pub fn code_and_link_spans(body: &str, format: ContentFormat) -> crate::error::Result<BodySpans> {
    let mut doc = parse(body, format)?;
    let nodes = doc
        .nodes()
        .map_err(|e| crate::error::Error::Content(format!("twig nodes: {e}")))?;
    let mut code = Vec::new();
    let mut links = Vec::new();
    let mut images = Vec::new();
    for node in nodes {
        if is_code(&node.kind) {
            code.push(node.span);
        } else if node.kind == twig::Kind::Link {
            links.push(node.span);
        } else if node.kind == twig::Kind::Image {
            images.push(node.span);
        }
    }
    code.sort_by_key(|s: &Range<usize>| s.start);
    links.sort_by_key(|s: &Range<usize>| s.start);
    images.sort_by_key(|s: &Range<usize>| s.start);
    Ok(BodySpans {
        code,
        links,
        images,
    })
}

/// The spans of every node in `body` whose kind satisfies `want`, sorted by
/// start offset. The one walk [`code_spans`] and [`link_spans`] share.
///
/// **Why the flat node array and not `twig`'s selector query.** A parsed
/// document is not one tree. Footnote and link-reference definitions resolve by
/// *label* rather than by position, so twig attaches them to no parent — and
/// `twig_document_query` walks from the root, which means it never enters them.
/// Both of prov's uses were wrong inside a footnote in the two opposite
/// directions at once: a `` `[[x]]` `` in a footnote body was not reported as
/// code, so the lexical wikilink scan promoted it to a link (DESIGN §8's
/// false positive, the exact thing [`code_spans`] exists to prevent), and a real
/// `[a](b.md)` in a footnote body was not reported as a link, so a rename never
/// rewrote it and no broken-link finding was ever raised for it.
///
/// `Document::nodes` is indexed over the whole arena rather than walked from the
/// root, so the detached definition subtrees are simply in it. It also replaces
/// the four separate queries [`code_spans`] used to issue — twig's selector
/// grammar has no union combinator, and a kind predicate needs none.
fn spans_where(
    body: &str,
    format: ContentFormat,
    want: impl Fn(&twig::Kind) -> bool,
) -> crate::error::Result<Vec<std::ops::Range<usize>>> {
    let mut doc = parse(body, format)?;
    let nodes = doc
        .nodes()
        .map_err(|e| crate::error::Error::Content(format!("twig nodes: {e}")))?;
    let mut spans: Vec<_> = nodes
        .into_iter()
        .filter(|n| want(&n.kind))
        .map(|n| n.span)
        .collect();
    spans.sort_by_key(|s| s.start);
    Ok(spans)
}

/// The byte ranges in `body` that `twig` parses as code (inline code spans,
/// fenced code blocks, raw inline/block escapes) — everything a link scan
/// should treat as opaque; see [`crate::link::scan_wikilinks`]. Spans are
/// returned sorted by start offset, and cover code inside a footnote definition
/// as well as code in the document body (see `spans_where`).
pub fn code_spans(
    body: &str,
    format: ContentFormat,
) -> crate::error::Result<Vec<std::ops::Range<usize>>> {
    spans_where(body, format, is_code)
}

/// The byte ranges in `body` that `twig` parses as inline links — the whole
/// `[text](target)` construct of each `link` node, in source order. This is the
/// syntax-aware, code-aware complement to prov's lexical `[[…]]` scan: twig
/// never reports a `[x](y)` inside a code fence, an autolink's angle brackets,
/// or bracket text that is not actually a link, so a body-link scan built on
/// these spans cannot mistake prose or code for a link. The caller slices each
/// span and parses it with [`crate::link::Link::parse`] to read the target —
/// each span holds exactly one link, so the parse never over-reaches.
///
/// Reference-style and autolink forms also surface as `link` nodes; the caller
/// keeps only the inline `[label](target)` ones (a successful markdown parse),
/// which is the form prov can resolve and rewrite in place.
///
/// Links inside a footnote definition are included, which a walk from the
/// document root does not reach — see `spans_where`.
pub fn link_spans(
    body: &str,
    format: ContentFormat,
) -> crate::error::Result<Vec<std::ops::Range<usize>>> {
    spans_where(body, format, |k| *k == twig::Kind::Link)
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn infers_format_from_extension() {
        assert_eq!(
            ContentFormat::from_extension(Path::new("a.md")),
            Some(ContentFormat::Markdown)
        );
        assert_eq!(
            ContentFormat::from_extension(Path::new("a.markdown")),
            Some(ContentFormat::Markdown)
        );
        assert_eq!(
            ContentFormat::from_extension(Path::new("a.dj")),
            Some(ContentFormat::Djot)
        );
        assert_eq!(
            ContentFormat::from_extension(Path::new("a.djot")),
            Some(ContentFormat::Djot)
        );
        assert_eq!(
            ContentFormat::from_extension(Path::new("a.html")),
            Some(ContentFormat::Html)
        );
        assert_eq!(
            ContentFormat::from_extension(Path::new("a.htm")),
            Some(ContentFormat::Html)
        );
        assert_eq!(ContentFormat::from_extension(Path::new("a.yaml")), None);
        assert_eq!(ContentFormat::from_extension(Path::new("noext")), None);
    }

    #[test]
    fn extension_matches_the_primary_from_extension_spelling() {
        // The derived filename's extension must round-trip back to the same format.
        for f in [
            ContentFormat::Markdown,
            ContentFormat::Djot,
            ContentFormat::Html,
        ] {
            let name = format!("derived.{}", f.extension());
            assert_eq!(ContentFormat::from_extension(Path::new(&name)), Some(f));
        }
    }

    #[test]
    fn transcode_respells_the_grammar_and_leaves_a_matching_body_alone() {
        // Markdown → Djot is a genuine re-spelling, not a copy: setext headings
        // become ATX and emphasis takes djot's markers. `[[wikilinks]]` are prov's
        // notation rather than twig's, and must ride through untouched — the
        // `convert` engine renames files on this promise.
        let md = "Title\n=====\n\n*emph* and [[a:b]] and `code`.\n";
        let dj = transcode(md, ContentFormat::Markdown, ContentFormat::Djot).unwrap();
        assert!(dj.contains("# Title"), "{dj}");
        assert!(dj.contains("_emph_"), "{dj}");
        assert!(dj.contains("[[a:b]]"), "wikilink verbatim: {dj}");
        assert!(dj.contains("`code`"), "{dj}");

        // Same grammar in and out: returned verbatim, so prose the author wrapped
        // by hand is never silently reflowed.
        assert_eq!(
            transcode(md, ContentFormat::Markdown, ContentFormat::Markdown).unwrap(),
            md
        );

        // And the source grammar is honoured, not assumed: djot's `_emph_` read as
        // djot survives, where reading it as Markdown would not.
        let back = transcode(&dj, ContentFormat::Djot, ContentFormat::Markdown).unwrap();
        assert!(back.contains("*emph*"), "djot read as djot: {back}");
    }

    #[test]
    fn html_is_the_lossy_endpoint_in_both_directions() {
        // What `convert --force` gates: the prose grammars interconvert freely,
        // and every pairing with HTML is lossy whichever way it runs.
        assert!(!ContentFormat::Markdown.is_lossy_to(ContentFormat::Djot));
        assert!(!ContentFormat::Djot.is_lossy_to(ContentFormat::Markdown));
        assert!(ContentFormat::Markdown.is_lossy_to(ContentFormat::Html));
        assert!(ContentFormat::Html.is_lossy_to(ContentFormat::Djot));
        // A conversion to the grammar already in use is not a conversion at all.
        for f in [
            ContentFormat::Markdown,
            ContentFormat::Djot,
            ContentFormat::Html,
        ] {
            assert!(!f.is_lossy_to(f));
        }
    }

    #[test]
    fn code_and_link_spans_reach_inside_a_footnote_definition() {
        // A footnote definition resolves by label, so twig attaches it to no
        // parent and a walk from the document root never enters it. Both spans
        // are read out of the flat node array for exactly this case.
        //
        // The code half is DESIGN §8's false positive: unmasked, the lexical
        // `[[…]]` scan promotes this code to a link.
        for format in [ContentFormat::Markdown, ContentFormat::Djot] {
            let body = "Body.[^n]\n\n[^n]: A note with `[[inf] * n]` code.\n";
            let spans = code_spans(body, format).unwrap();
            assert_eq!(
                spans.iter().map(|s| &body[s.clone()]).collect::<Vec<_>>(),
                ["`[[inf] * n]`"],
                "{format:?}"
            );

            // The link half: unreported, a rename never rewrites it and no
            // broken-link finding is ever raised for it.
            let body = "Body.[^n]\n\n[^n]: See [b](notes/b.md) here.\n";
            let spans = link_spans(body, format).unwrap();
            assert_eq!(
                spans.iter().map(|s| &body[s.clone()]).collect::<Vec<_>>(),
                ["[b](notes/b.md)"],
                "{format:?}"
            );
        }
    }

    /// The combined accessor exists only to save a parse, so it must answer
    /// exactly what the two single-kind ones do — including the sort order and
    /// the footnote-definition subtrees a walk from the root would miss.
    #[test]
    fn one_parse_reports_what_the_two_separate_ones_do() {
        let body = concat!(
            "A [real](a.md) link and `[[code]]` and a [second](b.md).\n\n",
            "```\n[fenced](c.md)\n```\n\n",
            "Text[^fn]\n\n[^fn]: a note with [a link](d.md) and `code`.\n",
        );
        let spans = code_and_link_spans(body, ContentFormat::Markdown).unwrap();
        assert_eq!(
            spans.code,
            code_spans(body, ContentFormat::Markdown).unwrap()
        );
        assert_eq!(
            spans.links,
            link_spans(body, ContentFormat::Markdown).unwrap()
        );
        assert!(
            !spans.code.is_empty() && !spans.links.is_empty(),
            "fixture found nothing"
        );
    }

    #[test]
    fn image_spans_cover_the_whole_construct_in_both_grammars() {
        // The `!` is in the span, an empty alt is still an image, a footnote
        // definition is reached, and an image in a code span is code.
        for format in [ContentFormat::Markdown, ContentFormat::Djot] {
            let body = "![A photo](a.jpg) and ![](b.jpg) and `![c](c.jpg)`.[^n]\n\n\
                        [^n]: ![n](n.png)\n";
            let spans = code_and_link_spans(body, format).unwrap();
            assert_eq!(
                spans
                    .images
                    .iter()
                    .map(|s| &body[s.clone()])
                    .collect::<Vec<_>>(),
                ["![A photo](a.jpg)", "![](b.jpg)", "![n](n.png)"],
                "{format:?}"
            );
            assert!(spans.links.is_empty(), "{format:?}: an image is not a link");
        }
    }

    #[test]
    fn code_spans_covers_every_code_bearing_kind_in_one_pass() {
        // One kind predicate over the flat array replaces four separate
        // selector queries; all four kinds must still be reported, in source
        // order regardless of the order the arena holds them in.
        let body = "`v`\n\n```\nfenced\n```\n\n<span>raw</span>\n\n<div>\nblock\n</div>\n";
        let spans = code_spans(body, ContentFormat::Markdown).unwrap();
        assert!(
            spans.windows(2).all(|w| w[0].start <= w[1].start),
            "sorted by start: {spans:?}"
        );
        let covered: String = spans.iter().map(|s| &body[s.clone()]).collect();
        // One fixture per kind: verbatim, code_block, raw_inline, raw_block.
        // A raw *inline* is the tag alone — the prose between `<span>` and
        // `</span>` is not code and is deliberately left unmasked — whereas a
        // raw *block* is the whole element, its interior included.
        for expected in ["`v`", "fenced", "<span>", "<div>\nblock\n</div>"] {
            assert!(
                covered.contains(expected),
                "{expected} missing: {covered:?}"
            );
        }
    }

    #[test]
    fn renders_markdown_to_html_via_twig_ffi() {
        let html = render_html("# hi\n", ContentFormat::Markdown).unwrap();
        assert_eq!(html, "<h1>hi</h1>\n");
    }

    #[test]
    fn renders_djot_to_html_via_twig_ffi() {
        let html = render_html("_hi_\n", ContentFormat::Djot).unwrap();
        assert_eq!(html, "<p><em>hi</em></p>\n");
    }

    #[test]
    fn renders_html_via_twig_ffi() {
        let html = render_html("<p>hi</p>", ContentFormat::Html).unwrap();
        assert!(html.contains("hi"));
    }

    #[test]
    fn link_spans_find_inline_links_but_not_code_or_prose() {
        let body = "See [the doc](notes/a.md) and `[not](a link)` and plain [text].";
        let spans = link_spans(body, ContentFormat::Markdown).unwrap();
        // The real inline link is found, its span the whole `[label](target)`.
        let want = body.find("[the doc](notes/a.md)").unwrap();
        assert!(
            spans
                .iter()
                .any(|s| s.start == want && &body[s.clone()] == "[the doc](notes/a.md)"),
            "expected the inline link span, got {spans:?}"
        );
        // The backtick-wrapped `[not](a link)` is code, not a link.
        let code_at = body.find("[not]").unwrap();
        assert!(
            !spans.iter().any(|s| s.contains(&code_at)),
            "code must not be a link: {spans:?}"
        );
        // `[text]` with no destination is not a link either.
        let bracket_at = body.find("[text]").unwrap();
        assert!(
            !spans.iter().any(|s| s.contains(&bracket_at)),
            "bare brackets are not a link"
        );
    }

    #[test]
    fn link_spans_read_djot_links_too() {
        let body = "Here is [a link](../b.dj) inline.\n";
        let spans = link_spans(body, ContentFormat::Djot).unwrap();
        assert!(
            spans
                .iter()
                .any(|s| &body[s.clone()] == "[a link](../b.dj)"),
            "djot inline link span, got {spans:?}"
        );
    }

    #[test]
    fn code_spans_cover_verbatim_but_not_prose() {
        let body = "See [[colophon:abc123]] and `[[inf] * n for _ in range(m)]]` here.";
        let spans = code_spans(body, ContentFormat::Markdown).unwrap();

        // The plain wikilink is untouched by any code span...
        let wikilink_span = body.find("[[colophon:abc123]]").unwrap()
            ..body.find("[[colophon:abc123]]").unwrap() + "[[colophon:abc123]]".len();
        assert!(
            !spans
                .iter()
                .any(|cs| cs.start < wikilink_span.end && wikilink_span.start < cs.end)
        );

        // ...but the backtick-wrapped one is inside exactly one code span.
        let code_start = body.find('`').unwrap();
        let code_end = body.rfind('`').unwrap() + 1;
        assert!(
            spans
                .iter()
                .any(|cs| cs.start <= code_start && code_end <= cs.end)
        );
    }
}