quillmark_core/document/

mod.rs

//! Parsing and typed in-memory model for Quillmark card-yaml documents.
//!
//! A [`Document`] holds a root [`Card`] plus ordered composable cards; each
//! card carries a [`Payload`], source-ordered items ([`PayloadItem`]:
//! `$quill`/`$kind`/`$id` metadata, user fields, and comments, in the order
//! they appear in the block's YAML content), and a Markdown body.
//! [`Document::parse`] returns errors for malformed YAML, unclosed
//! fences, a missing root `$quill`, or unknown `$`-prefixed system keys.
//!
//! See [markdown-spec.md](https://github.com/borb-sh/quillmark/blob/main/prose/references/markdown-spec.md)
//! for the card-yaml format specification.

use serde::{Deserialize, Serialize};

use quillmark_content::import::{from_markdown as import_markdown, ImportError};
use quillmark_content::Content;

use crate::error::ParseError;
use crate::version::QuillReference;
use crate::Diagnostic;

/// The single markdown→content boundary for card bodies. Every construction path
/// that starts from an authored markdown string ([`Document::parse`],
/// wire/storage deserialization, seeding, blueprint) routes through it, so the
/// markdown parser is reached from exactly one helper. An empty string yields
/// the empty content without invoking the parser.
pub(crate) fn import_body(md: &str) -> Result<Content, ImportError> {
    if md.is_empty() {
        Ok(Content::empty())
    } else {
        import_markdown(md)
    }
}

/// Which encoding a `decode_richtext_value` failure came from, so a call site
/// can prefix its diagnostic per encoding without re-deriving the dispatch.
/// Surfaced publicly as the error of [`Card::field_richtext`].
#[derive(Debug, Clone, PartialEq, Eq)]
#[non_exhaustive]
pub enum RichtextDecodeError {
    /// A JSON object that is not a valid canonical content.
    NotContent(String),
    /// A markdown string that failed to import.
    BadMarkdown(String),
}

impl RichtextDecodeError {
    /// The inner failure message, without an encoding-specific prefix.
    pub fn into_message(self) -> String {
        match self {
            RichtextDecodeError::NotContent(m) | RichtextDecodeError::BadMarkdown(m) => m,
        }
    }
}

/// Decode a JSON value in either accepted richtext encoding: a canonical content
/// **object** ([`from_canonical_value`](quillmark_content::serial::from_canonical_value))
/// or an authored markdown **string** (via [`import_body`], the single markdown
/// boundary). The one place the object-vs-string dispatch lives; a call site
/// handles the shapes that are neither (`null`, array, scalar) and maps the
/// error into its own type.
///
/// - `Some(Ok(rt))`: decoded.
/// - `Some(Err(e))`: an object that is not a content, or a string that failed
///   to import; `e` names the encoding so the caller can prefix its message.
/// - `None`: the value is neither an object nor a string.
pub(crate) fn decode_richtext_value(
    value: &serde_json::Value,
) -> Option<Result<Content, RichtextDecodeError>> {
    match value {
        serde_json::Value::Object(_) => Some(
            quillmark_content::serial::from_canonical_value(value)
                .map_err(|e| RichtextDecodeError::NotContent(e.to_string())),
        ),
        serde_json::Value::String(md) => {
            Some(import_body(md).map_err(|e| RichtextDecodeError::BadMarkdown(e.to_string())))
        }
        _ => None,
    }
}

/// Decode a JSON value for a `plaintext` field: a canonical content **object**
/// (revalidated) or a literal **string** imported verbatim
/// ([`from_plaintext`](quillmark_content::from_plaintext), never markdown, so
/// `*hi*` stays four plain characters). The plaintext twin of
/// [`decode_richtext_value`]: the string branch is infallible (literal import
/// can't fail), so only the object branch yields `Err` (`String` message). A
/// call site handles the shapes that are neither: `null`, array, scalar. This
/// is the single plaintext object-vs-string dispatch, shared by the coercion
/// literal-import site and the validation shape check.
///
/// - `Some(Ok(rt))`: decoded.
/// - `Some(Err(msg))`: an object that is not a valid content.
/// - `None`: the value is neither an object nor a string.
pub(crate) fn decode_plaintext_value(
    value: &serde_json::Value,
) -> Option<Result<Content, String>> {
    match value {
        serde_json::Value::Object(_) => {
            Some(quillmark_content::serial::from_canonical_value(value).map_err(|e| e.to_string()))
        }
        serde_json::Value::String(s) => Some(Ok(quillmark_content::from_plaintext(s))),
        _ => None,
    }
}

pub mod assemble;
pub mod dto;
pub mod edit;
pub mod emit;
pub mod fences;
pub mod limits;
pub mod meta;
pub mod payload;
pub(crate) mod prescan;
pub mod wire;
pub(crate) mod yaml_hints;

pub use dto::{peek_schema_version, StorageError, StoredDocument, SCHEMA_V0_93_0};
pub use edit::EditError;
pub use meta::{is_valid_kind_name, validate_composable_kind, CardKindError};
pub use payload::{MetaKey, Payload, PayloadItem};
// Reachable through `PayloadItem::nested_comments`, so nameable from here.
pub use prescan::{CommentPathSegment, NestedComment};
pub use wire::{CardWire, PayloadItemWire, WireError};

/// Authoring-format rules for the `~~~` card-yaml markdown surface.
///
/// Surfaced verbatim to LLM/MCP consumers (and to CLI / Python bindings via
/// the same text) so error parity holds: every consumer reads the same
/// rules. This is the single source of truth; bindings should call into it
/// rather than re-stating the rules in their own glue.
pub const FORMAT_RULES: &str = "Document format rules:
\u{2022} Block opener and closer are EXACTLY `~~~` (three tildes, no info string). The `~~~card-yaml` opener is also accepted as a non-canonical alias.
\u{2022} A blank line must precede every `~~~` block opener (unless it is line 1), and the opener must be at column zero (no leading spaces). An indented `~~~` is an ordinary code block, not a card.
\u{2022} The first block is the root and MUST contain `$quill: <name>@<version>`. Its `$kind` is `main` by position \u{2014} an explicit `$kind: main` is accepted but not required. Additional blocks declare composable cards via `$kind: <card_kind>`.
\u{2022} Reserved `$`-keys: `$quill`, `$kind`, `$id`, `$ext`, `$seed`. User fields use lowercase snake_case.
\u{2022} Prose body is the text after a block's closing `~~~`, up to the next opener or EOF. To include a literal fenced code block in prose, use a backtick fence (```); any column-zero `~~~` block is parsed as card metadata.
\u{2022} A field that already shows a concrete value carries a default and is shippable as-is \u{2014} keep the line, override the value, or delete it to fall back to the default. A blank or null value (`field:`, `field: null`, `field: ~`) is treated the same as omitting the field: it falls back to the default, or to the type-empty zero value.
\u{2022} `field: !must_fill <value>` marks a placeholder awaiting your input \u{2014} replace it with a real value and drop the `!must_fill` tag before shipping. A bare `field: !must_fill` is an empty placeholder. A leftover marker never blocks rendering, but it is reported as a warning until you replace it.
\u{2022} Numbers and booleans MUST be unquoted (`year: 2025`, `pinned: true`); quoting turns them into strings and fails validation.
\u{2022} Plain-scalar values cannot start with `*` or `&` (YAML alias/anchor markers) and cannot contain `: ` (colon-space). For markdown emphasis, embedded colons, or other special prefixes, quote the value: `field: '**bold**'` or `field: \"Name: subtitle\"`. Multi-line values use `|-`, not multi-line quoted scalars.";

/// Authoring-ergonomics header that introduces a blueprint to an LLM/MCP
/// consumer. The `{quill}` placeholder is substituted with the quill name.
/// Designed to be shown above [`FORMAT_RULES`], which covers field-level
/// semantics like the `!must_fill` marker: keep the wording tight here so the
/// two strings do not duplicate guidance.
const BLUEPRINT_INSTRUCTION_TEMPLATE: &str =
    "Fill in the `{quill}` blueprint below: replace each `!must_fill` placeholder with a real \
value and edit the body prose. Submit the filled markdown as `content` to `create_document`.";

/// Render the blueprint-instruction header with `quill_name` substituted in.
/// Single source of truth for the prose so every binding shows identical text.
pub fn blueprint_instruction(quill_name: &str) -> String {
    BLUEPRINT_INSTRUCTION_TEMPLATE.replace("{quill}", quill_name)
}

#[cfg(test)]
mod tests;

/// The record of one parse: the [`Document`] and any non-fatal warnings.
/// Returned by [`Document::parse`], the single parse entry. Warnings live here
/// and only here: `Document` is the value (equality, the storage DTO, and
/// mutators all exclude warnings); `Parsed` is the parse *event*. A caller that
/// wants only the document writes `Document::parse(md)?.document`.
#[derive(Debug)]
#[must_use = "carries parse warnings; read `.document`/`.warnings` or bind it"]
#[non_exhaustive]
pub struct Parsed {
    pub document: Document,
    pub warnings: Vec<Diagnostic>,
}

/// A single card-yaml block (root or composable). `body` is the content
/// ([`Content`]) form of the prose after the closing fence: the empty content
/// when none follows; check `card.body().is_blank()`. Markdown is a projection:
/// [`Card::body_markdown`] re-emits it.
#[derive(Debug, Clone, PartialEq)]
pub struct Card {
    payload: Payload,
    body: Content,
}

impl Card {
    /// Create a `Card` from its parts without validation. `body` is the content
    /// form; to build from an authored markdown string, import it first via the
    /// crate-internal `import_body` boundary. For user-facing construction of
    /// composable cards use [`Card::new`].
    pub fn from_parts(payload: Payload, body: Content) -> Self {
        Self { payload, body }
    }

    pub fn quill(&self) -> Option<&QuillReference> {
        self.payload.quill()
    }

    pub fn kind(&self) -> Option<&str> {
        self.payload.kind()
    }

    pub fn id(&self) -> Option<&str> {
        self.payload.id()
    }

    /// Opaque `$ext` map for out-of-band extension data (UI editor state,
    /// agent annotations, …). Carried through Markdown and storage DTO
    /// round-trips; never emitted into the plate JSON consumed by
    /// backends.
    pub fn ext(&self) -> Option<&serde_json::Map<String, serde_json::Value>> {
        self.payload.ext()
    }

    pub fn payload(&self) -> &Payload {
        &self.payload
    }

    pub fn payload_mut(&mut self) -> &mut Payload {
        &mut self.payload
    }

    /// The card body as a [`Content`] content: the canonical content model.
    /// For the markdown projection use [`Card::body_markdown`].
    pub fn body(&self) -> &Content {
        &self.body
    }

    /// The card body rendered back to its markdown projection. This is a
    /// derived view (`export ∘ body`), not stored state; a `Document` round-trip
    /// therefore canonicalizes the body (e.g. `__b__` → `**b**`).
    pub fn body_markdown(&self) -> String {
        quillmark_content::export::to_markdown(&self.body)
    }

    pub(crate) fn overwrite_body(&mut self, body: Content) {
        self.body = body;
    }

    pub(crate) fn body_mut(&mut self) -> &mut Content {
        &mut self.body
    }

    /// Read a richtext-valued user field back as a [`Content`] content: the
    /// field-level twin of [`Card::body`]. Decodes the stored value through the
    /// same object-or-markdown dispatch the writer
    /// ([`commit_field`](Card::commit_field)) commits, so a field
    /// stored as a canonical content reads back losslessly (identity marks
    /// intact) and a still-authored markdown string imports.
    ///
    /// - `None`: the field is absent.
    /// - `Some(Ok(rt))`: decoded content.
    /// - `Some(Err(_))`: the field is present but neither a content object nor
    ///   an importable markdown string (e.g. a bare number a `store_field` wrote).
    ///
    /// A `Document` carries no schema, so this cannot itself tell a richtext
    /// field from a plain string field; the caller names a field it knows is
    /// richtext, exactly as it does when writing.
    pub fn field_richtext(&self, name: &str) -> Option<Result<Content, RichtextDecodeError>> {
        let value = self.payload.get(name)?.as_json();
        Some(match crate::document::decode_richtext_value(value) {
            Some(result) => result,
            None => match value {
                serde_json::Value::Null => Ok(Content::empty()),
                _ => Err(RichtextDecodeError::NotContent(
                    "expected a richtext content object or a markdown string".to_string(),
                )),
            },
        })
    }

    /// The markdown projection of a richtext-valued field (`export ∘ decode`):
    /// the field-level twin of [`Card::body_markdown`], and the projection an
    /// emit or a markdown save writes for a content-valued field. The projection
    /// twin of [`field_richtext`](Card::field_richtext), carrying its `Ok`/`Err`
    /// decode outcome:
    ///
    /// - `None`: the field is absent.
    /// - `Some(Ok(md))`: the projected markdown.
    /// - `Some(Err(_))`: the field is present but does not decode as richtext
    ///   (a scalar/array/object a `store_field` wrote, or a non-content object).
    ///
    /// Absence returns `None`; a present non-richtext value returns `Some(Err)`,
    /// so the projection surfaces the type mismatch instead of blanking on it.
    pub fn field_markdown(&self, name: &str) -> Option<Result<String, RichtextDecodeError>> {
        Some(self.field_richtext(name)?.map(|rt| quillmark_content::export::to_markdown(&rt)))
    }

    /// The plaintext projection of a content-valued field (`to_plaintext ∘
    /// decode`), the literal-codec twin of [`field_markdown`](Card::field_markdown),
    /// for a `plaintext`-typed field: marks are never interpreted, so the text is
    /// verbatim both ways. Carries [`field_richtext`](Card::field_richtext)'s
    /// `Ok`/`Err` decode outcome:
    ///
    /// - `None`: the field is absent.
    /// - `Some(Ok(text))`: the projected literal text.
    /// - `Some(Err(_))`: the field is present but does not decode as content.
    pub fn field_plaintext(&self, name: &str) -> Option<Result<String, RichtextDecodeError>> {
        Some(self.field_richtext(name)?.map(|rt| quillmark_content::export::to_plaintext(&rt)))
    }
}

/// A parsed, per-kind **seed overlay**: the sparse fields (and optional body)
/// a newly-added card of a given kind starts with. Built from a `$seed[<kind>]`
/// entry of the main card's [`Card::seed`] map via [`SeedOverlay::from_json`],
/// and layered over the quill's schema-example seed by
/// [`crate::Quill::seed_card`] (overlay › example › absent). The reserved inner
/// key `$body` carries the body override; every other user field becomes an
/// entry, while any other `$`-prefixed key is reserved and dropped.
#[derive(Debug, Clone, PartialEq, Default)]
#[non_exhaustive]
pub struct SeedOverlay {
    /// Field-value overrides, keyed by field name.
    pub fields: indexmap::IndexMap<String, crate::value::QuillValue>,
    /// Body override, when the overlay declares a `$body` string.
    pub body: Option<String>,
}

impl SeedOverlay {
    /// Parse an overlay from a `$seed[<kind>]` JSON value, or `None` when it is
    /// not a mapping. Use this to turn the raw overlay object a consumer reads
    /// from the main card's `$seed` map ([`Card::seed`]) into a typed overlay to
    /// hand to [`crate::Quill::seed_card`]; e.g.
    /// `doc.main().seed().and_then(|m| m.get(kind)).and_then(SeedOverlay::from_json)`.
    pub fn from_json(value: &serde_json::Value) -> Option<Self> {
        value.as_object().map(Self::from_json_map)
    }

    /// Build an overlay from a single `$seed[<kind>]` JSON map: the reserved
    /// `$body` string becomes [`body`](Self::body); every other user-field entry
    /// becomes a field. A non-string `$body` is ignored (no body override). Any
    /// other `$`-prefixed key is reserved and dropped (never stored as a user
    /// field) since an overlay only ever carries user fields plus `$body`.
    fn from_json_map(map: &serde_json::Map<String, serde_json::Value>) -> Self {
        let mut fields = indexmap::IndexMap::new();
        let mut body = None;
        for (key, value) in map {
            if key == "$body" {
                if let Some(s) = value.as_str() {
                    body = Some(s.to_string());
                }
            } else if key.starts_with('$') {
                // Reserved key other than `$body`: not a user field. Drop it
                // rather than smuggle a `$`-key into the field set.
                continue;
            } else {
                fields.insert(
                    key.clone(),
                    crate::value::QuillValue::from_json(value.clone()),
                );
            }
        }
        SeedOverlay { fields, body }
    }
}

/// A fully-parsed Quillmark document. Serde routes through [`StoredDocument`];
/// for the plate wire shape see [`Document::to_plate_json`].
///
/// Parse-time warnings are *not* document state: they ride out-of-band on
/// [`Parsed`] from [`Document::parse`], the single owner. Equality and the
/// storage DTO therefore cover only structural content (`main` and `cards`).
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(into = "StoredDocument", try_from = "StoredDocument")]
pub struct Document {
    main: Card,
    cards: Vec<Card>,
}

impl Document {
    /// Create a blank document: a main card carrying only `$quill`, an empty
    /// body, and no composable cards. The programmatic blank canvas: every
    /// schema field is absent and resolves at render time (`default`, else
    /// type-empty zero), so nothing the caller did not set reaches the
    /// output. For an example-filled starter shaped like the blueprint, use
    /// `Quill::seed_document`.
    pub fn new(quill: QuillReference) -> Self {
        let mut payload = Payload::new();
        payload.set_quill(quill);
        // Parsed main cards always carry `$kind: main` (the parser normalizes
        // it in); match that shape so a blank document round-trips equal.
        payload.set_kind("main");
        Self {
            main: Card::from_parts(payload, Content::empty()),
            cards: Vec::new(),
        }
    }

    /// Create a `Document` from a pre-built main card and composable cards.
    /// `main` must carry `$quill`; composable cards must not.
    pub fn from_main_and_cards(main: Card, cards: Vec<Card>) -> Self {
        debug_assert!(main.quill().is_some(), "main card must carry `$quill`");
        debug_assert!(
            cards.iter().all(|c| c.quill().is_none()),
            "composable cards must not carry `$quill`"
        );
        debug_assert!(
            cards.iter().all(|c| c.seed().is_none()),
            "composable cards must not carry `$seed`"
        );
        debug_assert!(
            {
                let mut seen = std::collections::HashSet::new();
                cards
                    .iter()
                    .filter_map(|c| c.id())
                    .all(|id| !id.is_empty() && seen.insert(id))
            },
            "composable card `$id`s must be non-empty and unique per document"
        );
        Self { main, cards }
    }

    /// Parse card-yaml Markdown into a [`Parsed`]: the [`Document`] plus any
    /// non-fatal warnings. The single parse entry; a caller that wants only the
    /// document writes `Document::parse(md)?.document`. Errors on malformed
    /// YAML, a missing root `$quill`, an over-size input, and the other
    /// [`ParseError`] variants.
    #[doc(alias = "from_markdown")]
    pub fn parse(markdown: &str) -> Result<Parsed, ParseError> {
        assemble::decompose_with_warnings(markdown)
            .map(|(document, warnings)| Parsed { document, warnings })
    }

    pub fn main(&self) -> &Card {
        &self.main
    }

    pub fn main_mut(&mut self) -> &mut Card {
        &mut self.main
    }

    /// The `$quill` reference from the root block. Always present on parsed documents.
    pub fn quill_reference(&self) -> QuillReference {
        self.main
            .quill()
            .cloned()
            .expect("root block's $quill is validated at parse time")
    }

    pub fn cards(&self) -> &[Card] {
        &self.cards
    }

    pub fn cards_mut(&mut self) -> &mut [Card] {
        &mut self.cards
    }

    /// A single composable card by index: the immutable twin of
    /// [`card_mut`](Document::card_mut), so reading one card's payload does not
    /// require materializing every card via [`cards`](Document::cards). `None`
    /// when out of range.
    pub fn card(&self, index: usize) -> Option<&Card> {
        self.cards.get(index)
    }

    /// The composable card whose `$id` equals `id`, with its index:
    /// resolving the durable card handle ([PROGRAMMATIC.md]) without a
    /// hand-rolled scan over [`cards`](Document::cards). `$id` is unique per
    /// document (parse repairs a duplicate, mutators and storage reject one),
    /// so at most one card matches; `None` when none carries it.
    ///
    /// [PROGRAMMATIC.md]: https://github.com/borb-sh/quillmark/blob/main/prose/canon/PROGRAMMATIC.md
    pub fn find_card(&self, id: &str) -> Option<(usize, &Card)> {
        self.cards
            .iter()
            .enumerate()
            .find(|(_, card)| card.id() == Some(id))
    }

    pub(crate) fn cards_vec_mut(&mut self) -> &mut Vec<Card> {
        &mut self.cards
    }

    /// Serialize to the JSON wire shape consumed by backend plates. This is
    /// the **only** place in `quillmark-core` that produces this shape:
    ///
    /// ```json
    /// {
    ///   "$quill": "<ref>",
    ///   "$body": { "text": "…", "lines": [...], "marks": [...], "islands": [...] },
    ///   "$cards": [{ "$kind": "<tag>", "$body": <content>, "<field>": <value>, ... }],
    ///   "<field>": <value>, ...
    /// }
    /// ```
    ///
    /// `$body` (global and per-card) is canonical Content-JSON: the content as
    /// a nested object, not a markdown string. Richtext payload fields likewise
    /// cross as content objects (committed at coercion time).
    ///
    /// `$`-prefixed keys carry document-level metadata (quill ref, body
    /// text, card list, card kind). User payload fields stay flat at the
    /// root: they cannot collide with `$` keys because user field names are
    /// never `$`-prefixed (they match `[A-Za-z_][A-Za-z0-9_]*`).
    ///
    /// `$kind` is document-defined and omitted for a kindless card (never a
    /// fabricated `""`). This method is schema-free and emits `$body` for every
    /// card and the root; the schema-gated render plate
    /// (`QuillConfig::compile_data`) instead calls `to_plate_json_gated` with the
    /// per-card body-presence it resolved, so a card whose kind enables no body
    /// carries no `$body`: issue 1030's "absent on undefined".
    pub fn to_plate_json(&self) -> serde_json::Value {
        // Schema-free: the root and every card carry `$body`.
        self.to_plate_json_gated(true, None)
    }

    /// [`to_plate_json`](Self::to_plate_json) with the body-presence decision
    /// supplied by the caller: the root carries `$body` iff `main_body`, and card
    /// *i* iff `card_bodies` is `None` (all present) or `card_bodies[i]` holds.
    /// The schema-gated render plate (`QuillConfig::compile_data`) passes the
    /// body-enabled bit it already resolved per card, so a body-disabled card
    /// never carries `$body` (issue 1030, "absent on undefined") and the decision
    /// is never re-derived from the serialized plate. `Document` stays schema-free:
    /// it receives the decision, not a schema.
    pub(crate) fn to_plate_json_gated(
        &self,
        main_body: bool,
        card_bodies: Option<&[bool]>,
    ) -> serde_json::Value {
        let mut map = serde_json::Map::new();

        map.insert(
            "$quill".to_string(),
            serde_json::Value::String(self.quill_reference().to_string()),
        );

        // The seam carries the body as canonical Content-JSON (Option A): a
        // nested content object, byte-identical to `to_canonical_json`, never a lossy
        // markdown string. Backends lower the content (typst → markup + source
        // map; pdfform → `.text`); the markdown projection is `body_markdown`.
        if main_body {
            map.insert(
                "$body".to_string(),
                quillmark_content::serial::to_canonical_value(self.main.body()),
            );
        }

        let cards_array: Vec<serde_json::Value> = self
            .cards
            .iter()
            .enumerate()
            .map(|(i, card)| {
                let mut card_map = serde_json::Map::new();
                // A kindless card carries no `$kind`, never a fabricated `""`:
                // matching the resolved view's `kind: None`.
                if let Some(kind) = card.kind() {
                    card_map.insert(
                        "$kind".to_string(),
                        serde_json::Value::String(kind.to_string()),
                    );
                }
                // `$body` iff the caller's schema defines a body for this card.
                if card_bodies.map_or(true, |f| f.get(i).copied().unwrap_or(true)) {
                    card_map.insert(
                        "$body".to_string(),
                        quillmark_content::serial::to_canonical_value(card.body()),
                    );
                }
                for (key, value) in card.payload.iter() {
                    card_map.insert(key.clone(), value.as_json().clone());
                }
                serde_json::Value::Object(card_map)
            })
            .collect();

        map.insert("$cards".to_string(), serde_json::Value::Array(cards_array));

        for (key, value) in self.main.payload.iter() {
            map.insert(key.clone(), value.as_json().clone());
        }

        serde_json::Value::Object(map)
    }
}