rto-spec 5.7.0

//! Reading the **authored layer** out of a git tree: which files carry authored
//! intent, and what parsing each of them yields.
//!
//! This is one function ([`authored_layer`]) rather than a rule each caller
//! applies for itself, and that is the whole reason the module exists. The
//! authored file set must match the tree the *derived* layer was built from, and
//! the two disagreeing is issue #330 — a silent wrong answer, not a loud one.
//! [`rto_graph::GraphSource`] names the tree for both halves; this function is
//! the authored half of that pairing.
//!
//! It reads a git tree and parses text. It touches no [`Store`](rto_graph::Store)
//! and writes nothing, so a read-only surface can call it — which is what
//! [`crate::tool_check`] does.

use rto_graph::{BlobRef, GitError, GraphSource, Repo};

use crate::adr::AdrDoc;
use crate::annotate::Annotation;
use crate::blueprint::BlueprintDoc;
use crate::check::{Violation, ViolationKind};
use crate::site::SitePage;

/// Yields a blob's authored bytes, or `None` when the tree has no such file (a
/// worktree deletion, which the caller drops).
///
/// Generic over the error so a caller in a crate with its own error type — the
/// `roteiro` binary's `anyhow`, this crate's [`GitError`] — passes its own
/// closure without converting on the way in.
pub type BlobReader<'a, E> = dyn Fn(&BlobRef) -> Result<Option<Vec<u8>>, E> + 'a;

/// The authored documents found in one tree, ready for [`crate::check::run`] or
/// [`crate::check::validate`].
#[derive(Debug, Default)]
pub struct AuthoredLayer {
    /// ADRs under `docs/adr/` that parsed.
    pub docs: Vec<AdrDoc>,
    /// House-style blueprints (markdown, no frontmatter).
    pub blueprints: Vec<BlueprintDoc>,
    /// `@rto:` annotations scanned from every other file.
    pub annotations: Vec<Annotation>,
    /// ADRs under `docs/adr/` — and site pages anywhere — that did **not**
    /// parse. Carried as violations rather than dropped: a malformed ADR is
    /// drift, not a skippable warning — swallowing it lets the gate pass while
    /// silently discarding authored intent. A site page that declared itself
    /// published and then failed to parse is the same failure with a public
    /// consequence: the page silently does not exist.
    pub malformed: Vec<Violation>,
    /// House-style convention breaches found while reading the same blobs — see
    /// [`crate::convention`].
    ///
    /// Carried beside [`Self::malformed`] rather than inside it because the two
    /// are different claims: `malformed` is *this document does not parse*, and
    /// this is *this source breaks a rule we wrote down*. A caller that wants
    /// only parse failures should not have to filter prose to get them.
    pub conventions: Vec<Violation>,
}

/// Which files in `source`'s tree carry the authored layer.
///
/// The staged files in `Index` mode (so a staged-new ADR is seen), the `HEAD`
/// tree in `Committed` mode, and in `Worktree` mode `HEAD` **plus everything a
/// commit would add** — [`rto_graph::Repo::added_since_head`], which is
/// `untracked ∪ staged-but-not-in-HEAD`. That is precisely what
/// [`rto_graph::sync_worktree`] overlays into the derived layer, and the two must
/// describe the same tree.
///
/// It said "plus **untracked** files" until issue #657, and that one word was
/// wrong: `untracked_files` classifies against the *index*, so `git add` took a
/// new ADR out of the set without putting it into `HEAD`, and the union had a
/// hole exactly the size of "staged, not yet committed". Restated here because a
/// doc describing the old union is an invitation to reconstruct it.
///
/// Getting this wrong is issue #330's observed symptom. `sync_worktree` walks
/// untracked files deliberately, "so the working-tree `sync`/`check`/`review` see
/// new work that isn't staged yet" — but the authored set read only `HEAD`, so a
/// brand-new ADR had its symbols extracted while the file was never parsed as an
/// ADR. `check` then reported 17 ADRs with 18 on disk, `sync` said "up to date",
/// and nothing indicated that the newest decision was missing. The two layers
/// disagreed about which tree they were describing, in one worktree, with no
/// second worktree involved.
///
/// # Errors
/// Returns [`GitError`] if the tree or the index cannot be walked.
pub fn authored_blobs(repo: &Repo, source: GraphSource) -> Result<Vec<BlobRef>, GitError> {
    match source {
        GraphSource::Index => repo.index_files(),
        GraphSource::Committed => repo.walk_blobs(),
        GraphSource::Worktree => {
            let mut blobs = repo.walk_blobs()?;
            // Every path a commit would add, staged or not — **not**
            // `untracked_files` alone, which was issue #657: that set is defined
            // against the **index**, so `git add` on a new ADR took it out of the
            // untracked set without putting it into HEAD, and the authored layer
            // stopped seeing the file. `check` then reported `0 violations` on
            // drift it had named one `git add` earlier. The old comment here was
            // right that the two sets cannot overlap, and that is exactly why it
            // did not notice the union had a hole.
            //
            // The synthesized oid is unused: `Repo::read_source` reads Worktree
            // content from disk by path, and a not-yet-committed file has no HEAD
            // object to read anyway. (A bare repo has no working tree, so this
            // set is empty there and the oid-reading fallback is never reached
            // with one of these.)
            let head_paths: std::collections::BTreeSet<&str> =
                blobs.iter().map(|b| b.path.as_str()).collect();
            let added = repo.added_since_head(&head_paths)?;
            blobs.extend(added.into_iter().map(|path| BlobRef {
                path,
                oid: String::new(),
            }));
            Ok(blobs)
        }
    }
}

/// The authored layer **plus the site pages** — everything one tree's classify
/// pass yields.
///
/// A wrapper rather than a fourth field on [`AuthoredLayer`], because that struct
/// is destructured exhaustively by its callers and a new field is a breaking
/// change for every one of them. Wrapping lets a caller adopt site pages when it
/// is ready to render and gate them, and lets the rest keep compiling against
/// exactly the layer they already handle — which matters here because the
/// classification below must stay the *one* copy of the rule either way.
#[derive(Debug, Default)]
pub struct AuthoredDocs {
    /// ADRs, blueprints, annotations, and anything malformed — including a site
    /// page that declared itself published and then failed to parse.
    pub layer: AuthoredLayer,
    /// Documents that declared themselves published (`site-page:` frontmatter).
    pub site: Vec<SitePage>,
}

/// Classify and parse the authored layer out of `blobs`, reading each blob's
/// bytes with `read` — **discarding the site pages**.
///
/// Site pages are classified (they are not ADRs, blueprints or annotation
/// carriers, and misfiling them would put website prose into the annotation
/// scan) and then dropped, because this function's return type has nowhere to
/// put them. A caller that publishes or gates the website wants
/// [`authored_docs_from`], which is this function's whole body with the site
/// pages kept.
///
/// # Errors
/// Returns `E` if `read` fails.
pub fn authored_layer_from<E>(
    blobs: Vec<BlobRef>,
    read: &BlobReader<'_, E>,
) -> Result<AuthoredLayer, E> {
    Ok(authored_docs_from(blobs, read)?.layer)
}

/// Read and parse the authored layer from `source`'s tree, **discarding the site
/// pages** — see [`authored_layer_from`].
///
/// # Errors
/// Returns [`GitError`] if the tree cannot be walked or a source file cannot be
/// read.
pub fn authored_layer(repo: &Repo, source: GraphSource) -> Result<AuthoredLayer, GitError> {
    Ok(authored_docs(repo, source)?.layer)
}

/// Read and parse **everything** the authored classification yields from
/// `source`'s tree: the file set from [`authored_blobs`], the bytes from
/// [`Repo::read_source`], and the classification from [`authored_docs_from`].
///
/// # Errors
/// Returns [`GitError`] if the tree cannot be walked or a source file cannot be
/// read.
pub fn authored_docs(repo: &Repo, source: GraphSource) -> Result<AuthoredDocs, GitError> {
    authored_docs_from(authored_blobs(repo, source)?, &|blob| {
        repo.read_source(blob, source)
    })
}

/// Classify and parse the authored layer out of `blobs`, reading each blob's
/// bytes with `read`.
///
/// # This is the one copy of the classification rule
///
/// Which path is an ADR, which markdown is a blueprint, which markdown declares
/// itself a published site page, which file merely carries `@rto:` annotations,
/// and that a malformed ADR is drift rather than a skippable warning — that is a
/// rule with one correct answer, and it now has three callers
/// that reach it by different routes:
///
/// - [`authored_layer`] below, from a [`GraphSource`] tree (`build_graph`);
/// - `build_graph_at_rev` in the `roteiro` binary, from an arbitrary rev's blobs
///   (the Stage 35b graph arm, which needs the ADRs *of the reviewed commit*);
/// - [`crate::tool_check`], read-only, which cannot use either of the first two
///   because both end in a write.
///
/// Copying the loop would leave them free to drift, which is the shape this
/// repository has closed repeatedly — `[debt] ignore` honoured on three surfaces
/// and not a fourth, `limit == 0` meaning two things across five endpoints. A
/// graph arm whose ADRs were classified by a slightly different rule than
/// `check`'s would be measuring its own reimplementation.
///
/// `read` yields a blob's authored bytes, or `None` when the tree has no such
/// file (a worktree deletion); the caller supplies it because *where* the bytes
/// come from is precisely what differs between a tree, a rev, and a read-only
/// query. It is generic over its error so a caller in a crate with its own error
/// type does not have to convert on the way in.
///
/// # Errors
/// Returns `E` if `read` fails. A file that reads but does not *parse* is not an
/// error: a malformed ADR lands in [`AuthoredLayer::malformed`] as a violation.
pub fn authored_docs_from<E>(
    blobs: Vec<BlobRef>,
    read: &BlobReader<'_, E>,
) -> Result<AuthoredDocs, E> {
    let mut out = AuthoredDocs::default();
    let layer = &mut out.layer;
    for blob in blobs {
        // Parse the authored source from the same tree the derived layer used.
        let Some(bytes) = read(&blob)? else {
            continue;
        };
        let text = String::from_utf8_lossy(&bytes);
        let file = std::path::Path::new(&blob.path);
        let is_md = file
            .extension()
            .and_then(|e| e.to_str())
            .is_some_and(|e| e.eq_ignore_ascii_case("md"));
        let name = file
            .file_name()
            .and_then(|n| n.to_str())
            .unwrap_or_default();
        // Either the path this repository uses, **or** the document saying so.
        // The declaration is what makes the authored layer usable on a repository
        // whose decisions live somewhere else; the path is kept so nothing here
        // changes, and so an ADR that forgot the key is still found where it
        // lives. Same shape as the site-page rule below, for the same reason.
        // The `README.md` exclusion belongs to the **path** rule, not to both. It
        // is there because `docs/adr/README.md` is this repository's index of
        // decisions rather than one of them — a fact about that file, inferred
        // from its name because nothing in it says otherwise. A document that
        // declares `type: adr` has said otherwise, wherever it sits and whatever
        // it is called, and a repository keeping its one decision in
        // `architecture/decisions/README.md` is entitled to be believed.
        //
        // Our own index is unaffected: it declares no `type`, so the path rule
        // still excludes it.
        let is_adr = is_md
            && ((blob.path.starts_with("docs/adr/") && name != "README.md")
                || crate::adr::declares_adr(&text));
        if is_adr {
            match crate::adr::parse_adr(&blob.path, &text) {
                Ok(doc) => layer.docs.push(doc),
                Err(e) => layer.malformed.push(Violation {
                    kind: ViolationKind::MalformedAdr,
                    message: format!("{}: cannot parse ADR: {e}", blob.path),
                }),
            }
        } else if is_md && crate::site::is_site_page(&text) {
            // A document that declares itself published (`site-page:`) authors
            // `[[…]]` links like an ADR, and is checked the same way — which is
            // the entire reason the class exists. Classified before the blueprint
            // rule so a published document is never demoted by a coincidence of
            // its path or its H1.
            match crate::site::parse_site_page(&blob.path, &text) {
                Ok(page) => out.site.push(page),
                Err(e) => layer.malformed.push(Violation {
                    kind: ViolationKind::MalformedSitePage,
                    message: format!("{}: cannot parse site page: {e}", blob.path),
                }),
            }
        } else if is_md && crate::blueprint::is_blueprint(&blob.path, &text) {
            // House-style blueprints (no frontmatter) author `[[…]]` links like
            // ADRs; their links are drift-checked against the derived graph too.
            layer
                .blueprints
                .push(crate::blueprint::parse_blueprint(&blob.path, &text));
        } else {
            layer
                .annotations
                .extend(crate::annotate::scan_annotations(&blob.path, &text));
            // The same text, read once, for the conventions that are written
            // down and were not enforced. Riding this pass rather than adding a
            // second walk of the tree: the blobs are already open.
            layer
                .conventions
                .extend(crate::convention::scan_unjustified_allows(
                    &blob.path, &text,
                ));
            layer
                .conventions
                .extend(crate::convention::scan_lossy_identity(&blob.path, &text));
        }
    }
    Ok(out)
}

#[cfg(test)]
mod tests {
    use super::authored_docs_from;
    use crate::check::ViolationKind;
    use rto_graph::BlobRef;

    /// Classify a set of `(path, text)` pairs through the one classification
    /// rule, reading bytes straight from the fixture.
    fn classify(files: &[(&str, &str)]) -> super::AuthoredDocs {
        let blobs: Vec<BlobRef> = files
            .iter()
            .map(|(path, _)| BlobRef {
                path: (*path).to_owned(),
                oid: String::new(),
            })
            .collect();
        authored_docs_from(blobs, &|blob: &BlobRef| -> Result<Option<Vec<u8>>, ()> {
            Ok(files
                .iter()
                .find(|(p, _)| *p == blob.path)
                .map(|(_, text)| text.as_bytes().to_vec()))
        })
        .expect("classify")
    }

    #[test]
    fn publication_is_a_declaration_and_survives_living_outside_docs_site() {
        // The rule that makes the class worth having: `docs/OFFLINE_SETUP.md`
        // gains a public page *in place*, and the internal working documents
        // beside it stay internal — neither outcome depends on a path.
        let layer = classify(&[
            (
                "docs/OFFLINE_SETUP.md",
                "---\nsite-page: offline-setup\n---\n\n# Offline setup\n",
            ),
            (
                "docs/REVIEW_CHECKLIST.md",
                "# Review checklist\n\nInternal.\n",
            ),
            (
                "docs/history/BUILD_PLAN_V2.md",
                "# Build Plan V2\n\nInternal.\n",
            ),
        ]);
        let published: Vec<&str> = layer.site.iter().map(|p| p.path.as_str()).collect();
        assert_eq!(published, ["docs/OFFLINE_SETUP.md"]);
        assert_eq!(layer.site[0].slug, "offline-setup");
        assert!(
            layer.layer.malformed.is_empty(),
            "{:?}",
            layer.layer.malformed
        );
    }

    /// **A decision record is found where another repository keeps it.**
    ///
    /// The authored layer classified ADRs by path — anything under `docs/adr/`.
    /// That is this repository's convention rather than a property of the
    /// document, so the drift gate, the version rules and the link checking were
    /// all unavailable to any repository that keeps its decisions somewhere
    /// else. Roteiro is meant to run on other people's repositories, and this is
    /// what stopped it doing so for their decisions.
    ///
    /// The declaration wins wherever the file sits; the path still works for a
    /// document that did not declare anything.
    #[test]
    fn an_adr_is_recognised_wherever_a_repository_keeps_it() {
        let layer = classify(&[
            // Somewhere else entirely, and it says what it is.
            (
                "architecture/decisions/0007-thing.md",
                "---\ntype: adr\nadr-id: \"0007\"\nstatus: Accepted\n---\n\n# ADR-0007: Thing\n",
            ),
            // The old rule still holds for a document that declares nothing.
            (
                "docs/adr/0008-other.md",
                "---\nadr-id: \"0008\"\nstatus: Accepted\n---\n\n# ADR-0008: Other\n",
            ),
            // And ordinary prose is still ordinary prose, wherever it lives.
            ("notes/thoughts.md", "# Just a note\n\nNothing declared.\n"),
        ]);
        let mut ids: Vec<&str> = layer
            .layer
            .docs
            .iter()
            .map(|d| d.meta.id.as_str())
            .collect();
        ids.sort_unstable();
        assert_eq!(
            ids,
            vec!["0007", "0008"],
            "the declared one is found outside `docs/adr/`, and the path rule \
             still holds; malformed: {:?}",
            layer.layer.malformed
        );
        assert!(
            layer.layer.malformed.is_empty(),
            "{:?}",
            layer.layer.malformed
        );
    }

    /// **A repository may keep its one decision in a `README.md`.**
    ///
    /// The name exclusion exists because `docs/adr/README.md` is this
    /// repository's *index* of decisions rather than one of them — inferred from
    /// its name because nothing in the file says otherwise. A document that
    /// declares `type: adr` has said otherwise, so applying the exclusion to
    /// declared ADRs too would have made one layout impossible for no reason.
    #[test]
    fn a_declared_adr_is_not_excluded_for_being_called_readme() {
        let layer = classify(&[
            (
                "architecture/decisions/README.md",
                "---\ntype: adr\nadr-id: \"0001\"\nstatus: Accepted\n---\n\n# ADR-0001: The one\n",
            ),
            // And this repository's own index, which declares nothing, is still
            // excluded by the path rule.
            (
                "docs/adr/README.md",
                "# Architecture Decision Records\n\nAn index.\n",
            ),
        ]);
        let ids: Vec<&str> = layer
            .layer
            .docs
            .iter()
            .map(|d| d.meta.id.as_str())
            .collect();
        assert_eq!(
            ids,
            vec!["0001"],
            "the declared README is a decision, the undeclared one is an index; \
             malformed: {:?}",
            layer.layer.malformed
        );
        assert!(
            layer.layer.malformed.is_empty(),
            "{:?}",
            layer.layer.malformed
        );
    }

    /// **Declaring `type: adr` without an `adr-id` is malformed, not ignored.**
    ///
    /// Classification and parsing are separate: a document that says it is a
    /// decision record and then is not should be reported, because silently
    /// skipping it is how a repository ends up with decisions the gate never
    /// sees. This is the cost of the wider rule, and it is the right cost.
    #[test]
    fn a_document_claiming_to_be_an_adr_without_an_id_is_reported() {
        let layer = classify(&[(
            "elsewhere/half-baked.md",
            "---\ntype: adr\nstatus: Accepted\n---\n\n# Not really\n",
        )]);
        assert!(layer.layer.docs.is_empty());
        // Which violation, not merely that there is one. Asserting the count
        // alone would pass if the document had been misclassified and reported
        // as a malformed *site page* — the failure this test exists to exclude
        // is precisely a classification error, so the class has to be checked.
        assert_eq!(
            layer.layer.malformed.len(),
            1,
            "it must be reported rather than skipped: {:?}",
            layer.layer.malformed
        );
        let violation = &layer.layer.malformed[0];
        assert_eq!(
            violation.kind,
            ViolationKind::MalformedAdr,
            "reported as a malformed ADR, which is what it claimed to be: {violation:?}"
        );
        assert!(
            violation.message.contains("elsewhere/half-baked.md"),
            "and the message names the file: {violation:?}"
        );
    }

    #[test]
    fn an_adr_is_still_an_adr_and_a_page_outranks_the_blueprint_rule() {
        // ADRs are recognised first and are published by their own mechanism.
        // A declared page under `docs/blueprint/` must not be demoted to a
        // blueprint by the coincidence of its path.
        let layer = classify(&[
            (
                "docs/adr/0001-x.md",
                "---\nadr-id: \"0001\"\nstatus: Accepted\nsite-page: sneaky\n---\n\n# ADR-0001\n",
            ),
            (
                "docs/blueprint/landing.md",
                "---\nsite-page: index\n---\n\n# Roteiro\n",
            ),
            (
                "docs/blueprint/roteiro.md",
                "# Roteiro — Technical Implementation Plan\n",
            ),
        ]);
        assert_eq!(layer.layer.docs.len(), 1, "the ADR is still an ADR");
        let pages: Vec<&str> = layer.site.iter().map(|p| p.slug.as_str()).collect();
        assert_eq!(pages, ["index"]);
        assert_eq!(layer.layer.blueprints.len(), 1);
        assert_eq!(layer.layer.blueprints[0].path, "docs/blueprint/roteiro.md");
    }

    #[test]
    fn a_page_that_declares_itself_and_fails_to_parse_is_drift_not_silence() {
        // It asked to be published. Dropping it would leave the gate green and
        // the page silently absent from the site.
        let layer = classify(&[("docs/site/x.md", "---\nsite-page: Not A Slug\n---\n\n# X\n")]);
        assert!(layer.site.is_empty());
        assert_eq!(layer.layer.malformed.len(), 1);
        assert_eq!(
            layer.layer.malformed[0].kind,
            crate::check::ViolationKind::MalformedSitePage
        );
        assert!(
            layer.layer.malformed[0].message.contains("docs/site/x.md"),
            "names the file: {}",
            layer.layer.malformed[0].message
        );
    }

    #[test]
    fn the_three_field_entry_point_drops_pages_rather_than_misfiling_them() {
        // `authored_layer_from` has nowhere to put a site page. Dropping it is
        // deliberate and documented; the failure to avoid is the *other* one —
        // a published page falling through to the annotation scan, which would
        // put website prose into the `@rto:` surface.
        let files = [(
            "docs/OFFLINE_SETUP.md",
            "---\nsite-page: offline-setup\n---\n\n# Offline setup\n\n// @rto:0001\n",
        )];
        let blobs = vec![BlobRef {
            path: files[0].0.to_owned(),
            oid: String::new(),
        }];
        let layer =
            super::authored_layer_from(blobs, &|_: &BlobRef| -> Result<Option<Vec<u8>>, ()> {
                Ok(Some(files[0].1.as_bytes().to_vec()))
            })
            .expect("classify");
        assert!(layer.docs.is_empty());
        assert!(layer.blueprints.is_empty());
        assert!(
            layer.annotations.is_empty(),
            "a published page is not an annotation carrier: {:?}",
            layer.annotations
        );
        // The full form keeps it.
        assert_eq!(classify(&files).site.len(), 1);
    }

    #[test]
    fn a_non_page_still_contributes_its_annotations() {
        // Adding a class must not steal files from the annotation scan.
        let layer = classify(&[("src/store.rs", "//! @rto:0001\n")]);
        assert!(layer.site.is_empty());
        assert_eq!(layer.layer.annotations.len(), 1);
    }
}