//! Embedded config extraction: pull a config document out of a host file
//! (e.g. YAML frontmatter inside markdown) and parse it correctly.
const std = @import("std");
const Allocator = std.mem.Allocator;
const Language = @import("languages/language.zig");
const Document = @import("document.zig");
const Span = @import("util/span.zig");
const build_options = @import("build_options");
const Embed = @This();
const Delimiter = struct {
/// The exact line bodies this delimiter matches — used only by the FIXED
/// match modes (`whole_line`/`line_trimmed`/`prefix`). The PARAMETRIC modes
/// (`fenced_open`/`frontmatter_open`/`script_open`) ignore `tokens`: they
/// parse a language tag out of the line and resolve it, carrying the expected
/// format in the mode payload instead.
tokens: []const []const u8 = &.{},
match: Match,
/// The exact text emitted for this delimiter when *synthesizing* a region
/// (`initRegion`/`retype`), when it differs from `tokens[0]`. Required for the
/// parametric opens, whose match logic reads a tag rather than a literal —
/// e.g. `fenced` emits ```` ```toml ````, `frontmatter` emits `---toml` (or a
/// bare `---` for YAML), `script` emits the full `<script type="…">` tag.
/// Null ⇒ emit `tokens[0]` verbatim.
literal: ?[]const u8 = null,
};
/// How a `Delimiter` decides whether a host line is that delimiter.
const Match = union(enum) {
/// The line, trimmed of trailing space/tab, equals a token exactly. Leading
/// whitespace is significant (a markdown fence must sit at column 0), so an
/// indented line never matches.
whole_line,
/// The line (untrimmed) starts with a token.
prefix,
/// The line, trimmed of leading AND trailing whitespace, equals a token
/// exactly — the indentation-tolerant `whole_line`, for delimiters that may
/// sit indented inside a host (e.g. a `</script>` close tag nested in a
/// `<head>`).
line_trimmed,
/// The line opens a ```` ```<lang> ```` fenced block whose `<lang>` resolves
/// (`formatFromLangTag`) to this exact format. A bare ```` ``` ```` (no tag)
/// is a close, not an open, and never matches.
fenced_open: InnerFormat,
/// The line opens a `---<lang>` markdown-frontmatter block whose `<lang>`
/// resolves to this format; a bare `---` (no tag) resolves to `.yaml`.
frontmatter_open: InnerFormat,
/// The line, trimmed both sides, is an HTML `<script …>` open tag (on its own
/// line) whose `type` attribute's MIME resolves (`formatFromScriptMime`) to
/// this format. Attribute order, quoting, and whitespace are tolerated; a
/// same-line block (`<script …>body</script>`) is deliberately not matched.
script_open: InnerFormat,
/// The line opens an HTML `<code>` block (optionally wrapped in `<pre>`) whose
/// `class` list carries a `language-<lang>` token resolving to this format.
/// See `parseCodeOpen`.
code_open: InnerFormat,
};
/// The format an archetype's content is written in. A parametric archetype
/// carries one of these as its parameter (`fenced`/`frontmatter`/`html_script`);
/// the blessed presets each pin one. Named so callers outside this file can name
/// it too — see `innerFormat`.
pub const InnerFormat = enum { yaml, json, fig, toml };
/// Resolve a `fenced`/`frontmatter` language tag (a code-fence info string's
/// first token, or a `---<tag>`) to a config format — or null when it is not one
/// this project understands, so a ```` ```python ```` code sample or a `---foo`
/// line is left alone rather than mistaken for embedded config. `yml`/`figl` are
/// accepted spellings of `yaml`/`fig`.
fn formatFromLangTag(tag: []const u8) ?InnerFormat {
const eq = std.ascii.eqlIgnoreCase;
if (eq(tag, "yaml") or eq(tag, "yml")) return .yaml;
if (eq(tag, "json")) return .json;
if (eq(tag, "toml")) return .toml;
if (eq(tag, "fig") or eq(tag, "figl")) return .fig;
return null;
}
/// Resolve an HTML `<script type>` MIME to a config format, or null. Accepts the
/// canonical `application/<lang>` plus a few established aliases (`ld+json` for
/// JSON-LD, `application/figl` for the fig authoring dialect, `x-yaml`/`text</script>` HTML data
// island — the web's typed inert "data block" (cf. JSON-LD). It sits
// MID-document (typically in `<head>`), so it is located by scanning.
.open = .{ .match = .{ .script_open = f }, .literal = scriptLiteral(f) },
.close = .{ .tokens = &.{"</script>"}, .match = .line_trimmed },
.location = .middle,
.inner = f,
},
.html_code => |f| .{
// `<pre><code class="language-<lang>">` … `</code></pre>` — a VISIBLE,
// highlighted code block that is also the authoritative config. Its
// content is HTML-entity-encoded, so it is the one archetype with a
// non-identity codec (span-aware: see `Codec`/`reencodeEdited`).
.open = .{ .match = .{ .code_open = f }, .literal = codeLiteral(f) },
.close = .{ .tokens = &.{ "</code></pre>", "</code>" }, .match = .line_trimmed },
.location = .middle,
.inner = f,
.codec = .html_entities,
},
// --- blessed presets (distinct fixed delimiters) ---
.semicolons_json => .{
.open = .{ .tokens = &.{";;;"}, .match = .whole_line },
.close = .{ .tokens = &.{";;;"}, .match = .whole_line },
.location = .start,
.inner = .json,
},
.plus_toml => .{
.open = .{ .tokens = &.{"+++"}, .match = .whole_line },
.close = .{ .tokens = &.{"+++"}, .match = .whole_line },
.location = .start,
.inner = .toml,
},
.endmatter_yaml => .{
.open = .{ .tokens = &.{"```endmatter"}, .match = .whole_line },
.close = .{ .tokens = &.{"```"}, .match = .whole_line },
.location = .end,
.inner = .yaml,
},
};
}
/// An archetypal "config embedded in a host file" pattern. Each value fixes both
/// *where* the config lives (the host's delimiter convention) and *what* inner
/// format it is.
///
/// The three PARAMETRIC families take the format as a payload — the delimiter
/// convention is fixed but reads the format from a language tag (a fence info
/// string, a `---<lang>` tag, or a `<script type>` MIME):
/// - `frontmatter` — `---<lang>` … `---`/`...` (bare `---` ⇒ YAML). The whole
/// markdown-frontmatter ecosystem plus the self-describing `---<lang>` form.
/// - `fenced` — ```` ```<lang> ```` … ```` ``` ```` labeled code block.
/// - `html_script` — `<script type="application/<lang>">` … `</script>`.
///
/// The remaining BLESSED presets are the popular, *nonparametric* conventions
/// whose delimiter is its own distinct token (a reader decodes it without a
/// tag): `;;;` (JSON), `+++` (TOML, Hugo/Zola), and the trailing ```` ```endmatter ````
/// block. Keeping only these named — and expressing everything else
/// parametrically — is what shrinks the surface while still making every
/// (container, format) pair reachable.
pub const Type = union(enum) {
/// `---<lang>` … `---`/`...`; a bare `---` (no tag) is YAML.
frontmatter: InnerFormat,
/// ```` ```<lang> ```` … ```` ``` ````.
fenced: InnerFormat,
/// `<script type="application/<lang>">` … `</script>` HTML data island.
html_script: InnerFormat,
/// `<pre><code class="language-<lang>">` … `</code></pre>` — a visible,
/// highlighted, entity-encoded code block that is also the source of truth.
html_code: InnerFormat,
/// `;;;` … `;;;` JSON frontmatter (blessed; distinct delimiter).
semicolons_json,
/// `+++` … `+++` TOML frontmatter — the Hugo/Zola convention (blessed).
plus_toml,
/// A trailing ```` ```endmatter ```` … ```` ``` ```` YAML block. For Stephen Deken.
endmatter_yaml,
};
/// A located region, in *outer-source* byte coordinates. The fence spans are
/// retained so an editor can splice a replacement into `content` while leaving
/// everything else byte-identical. `body` is the host text OUTSIDE the region —
/// the markdown the config is embedded in — computed archetype-aware: the suffix
/// after the close fence for a `.start` (frontmatter) region, the prefix before
/// the open fence for an `.end` (endmatter) region. It is the read-side twin of
/// the `content` slice (frontmatter vs. body) and the target of `replace_body`.
pub const Region = struct {
open_fence: Span,
content: Span,
close_fence: Span,
body: Span,
};
/// Extraction result. `source` is the borrowed *outer* file; `region` indexes
/// into it. `document`'s node spans are relative to the DECODED content — call
/// `outerSpan` to lift them back into outer-file coordinates (which, for a codec
/// archetype, routes through the decode provenance map). `decoded` holds the
/// content the parser actually saw: a borrow of `source` for identity archetypes,
/// an owned decoded buffer (+ map) for the entity-encoded `<code>` one.
pub const Embedded = struct {
source: []const u8,
type: Type,
region: Region,
document: Document,
decoded: Decoded,
pub fn deinit(self: Embedded, allocator: Allocator) void {
self.document.deinit(allocator);
self.decoded.deinit(allocator);
}
pub fn outerSpan(self: Embedded, s: Span) Span {
const base = self.region.content.start;
// Map decoded coordinates back to source via the provenance map (a plain
// identity for the borrow case, where `segs` is empty).
return Span.init(self.decoded.srcAt(s.start) + base, self.decoded.srcAt(s.end) + base);
}
};
/// One document within a multi-document YAML stream, located in *outer-source*
/// byte coordinates. `content` is the exact slice handed to the single-document
/// parser: it INCLUDES a leading `---` marker line (which the parser consumes)
/// but excludes a trailing `...`. `explicit` records whether the document opened
/// with a `---` marker (vs. a bare document at stream start or after `...`).
pub const StreamDoc = struct {
content: Span,
explicit: bool,
document: Document,
/// Lift a node span (relative to this document's content) into outer-file
/// coordinates — mirrors `Embedded.outerSpan`.
pub fn outerSpan(self: StreamDoc, s: Span) Span {
const base = self.content.start;
return Span.init(s.start + base, s.end + base);
}
};
/// A parsed multi-document YAML stream. `source` is the borrowed outer file;
/// each document's `content` indexes into it. The single-document parser only
/// ever sees one document at a time, so this never trips its multi-document
/// guard — the stream concept lives here, in the splitter, not in the parser.
pub const Stream = struct {
source: []const u8,
documents: []const StreamDoc,
pub fn deinit(self: Stream, allocator: Allocator) void {
for (self.documents) |d| d.document.deinit(allocator);
allocator.free(self.documents);
}
};
pub const Error = error{
/// No region of this archetype exists (plain markdown, no frontmatter).
/// Distinct from a region that exists but is malformed.
NotFound,
/// An opening delimiter with no matching close.
Unterminated,
};
/// Locate + parse the embedded document of type `t` in `source`. For a codec
/// archetype the content is decoded (with provenance) before parsing, and the
/// decoded buffer is owned by the returned `Embedded`.
pub fn extract(allocator: Allocator, source: []const u8, t: Type) !Embedded {
const a = archetypeOf(t);
const region = try locate(source, a);
const decoded = try decodeForParse(allocator, Span.of(u8, region.content, source), a.codec);
errdefer decoded.deinit(allocator);
const document = try parseSlice(allocator, decoded.text, a.inner);
return .{ .source = source, .type = t, .region = region, .document = document, .decoded = decoded };
}
/// Locate the region of type `t` in `source` without parsing its content.
/// Useful when a caller only needs the fence/content spans (e.g. to splice).
pub fn locateRegion(source: []const u8, t: Type) Error!Region {
return locate(source, archetypeOf(t));
}
/// Best-effort content sniffing for which embed archetype `source` uses — the
/// `Embed` counterpart to `Language.detect`. Recognizes an OPEN delimiter (not a
/// full `locate`, which also demands a matching close — an unterminated block
/// should still be *recognized* so the caller's `extract`/`locateRegion` surfaces
/// the real `error.Unterminated` rather than a misleading "nothing found").
///
/// The `.start` conventions are checked on the very first line (after a BOM): a
/// fenced ```` ```<lang> ````, a `---<lang>` (or bare `---`) frontmatter, or a
/// `;;;`/`+++` block — their leading tokens are mutually distinct, so order among
/// them can't change the result. The scanned conventions come last because they
/// need a whole-document sweep: the trailing ```` ```endmatter ```` fence, then an
/// HTML `<script type>` data island anywhere in the page. A fenced tag that is
/// not a known config language (```` ```python ````, ```` ```endmatter ````) resolves to
/// null here, so it falls through to the next candidate rather than being taken
/// for config.
pub fn detect(source: []const u8) ?Type {
var i: usize = 0;
if (std.mem.startsWith(u8, source, "\xEF\xBB\xBF")) i += 3; // UTF-8 BOM
// First-line (`.start`) conventions.
const eol = lineEnd(source, i);
const first = std.mem.trim(u8, std.mem.trimEnd(u8, source[i..eol], "\r\n"), " \t");
if (parseFencedOpen(first)) |f| return .{ .fenced = f };
if (parseFrontmatterOpen(first)) |f| return .{ .frontmatter = f };
if (std.mem.eql(u8, first, ";;;")) return .semicolons_json;
if (std.mem.eql(u8, first, "+++")) return .plus_toml;
// Scanned conventions.
if (scanForDelim(source, i, archetypeOf(.endmatter_yaml).open) != null) return .endmatter_yaml;
var line = i;
while (line < source.len) : (line = lineEnd(source, line)) {
const le = lineEnd(source, line);
const lt = std.mem.trim(u8, std.mem.trimEnd(u8, source[line..le], "\r\n"), " \t");
if (parseScriptOpen(lt)) |f| return .{ .html_script = f };
if (parseCodeOpen(lt)) |f| return .{ .html_code = f };
}
return null;
}
/// Whether `t`'s body (the host prose) sits BEFORE the open fence (endmatter)
/// rather than after the close fence (frontmatter). Lets a host-coordinate
/// editor pick the side to splice when replacing the body.
pub fn bodyIsBefore(t: Type) bool {
return archetypeOf(t).location == .end;
}
/// The format `t`'s content is written in. Lets a caller resolve the parser/
/// printer to use for an embed's content — e.g. a `get`-style command that
/// picked an archetype via `--embed` and needs to know what format that
/// implies, rather than trusting a possibly-unrelated file-extension guess.
pub fn innerFormat(t: Type) InnerFormat {
return archetypeOf(t).inner;
}
/// The exact text to emit for delimiter `d` when synthesizing a region — its
/// `literal` override, or `tokens[0]` when there is none. (`script_open`'s match
/// token is a MIME value, not the `<script …>` tag it must emit.)
fn delimLiteral(d: Delimiter) []const u8 {
return d.literal orelse d.tokens[0];
}
/// Built host for a source that has no region of type `t` — the create half of
/// "open or create".
pub const Initialized = struct { host: []u8, region: Region };
/// Synthesize a host containing an EMPTY region of type `t` around `source`,
/// which is assumed to have none. The fresh block is placed where the archetype
/// dictates — prepended for a `.start` (frontmatter) region, appended for an
/// `.end` (endmatter) region — with the original `source` becoming the region's
/// body. Its content is seeded with an empty inner document (nothing for YAML, an
/// empty object for JSON) so a subsequent insert/set lands the first key. No
/// blank line is inserted between fence and body, so the output matches the
/// hand-rolled `---\n…\n---\n{body}` shape. The returned `host` is caller-owned.
///
pub fn initRegion(allocator: Allocator, source: []const u8, t: Type) !Initialized {
const a = archetypeOf(t);
const open_tok = delimLiteral(a.open);
const close_tok = delimLiteral(a.close);
// The empty inner document seeded between the fences. JSON gets a trailing
// newline so the close fence stays on its own line after the flow-mapping
// insert (which preserves it); YAML's, fig's, and TOML's block inserts emit
// their own newline, so their empty content needs none. Those three all seed
// empty because an empty document is a valid empty map / empty root table
// (see `fig/parser.zig`'s `buildRoot`), so a subsequent set/insert lands the
// first key into it.
const seed: []const u8 = switch (a.inner) {
.yaml, .fig, .toml => "",
.json => "{}\n",
};
var host: std.ArrayList(u8) = .empty;
errdefer host.deinit(allocator);
if (a.location == .end) {
// [ body ][ \n? ][ open\n ][ seed ][ close\n ]
try host.appendSlice(allocator, source);
// The open fence must start its own line; add a separating newline when
// the body doesn't already end in one.
if (source.len > 0 and source[source.len - 1] != '\n') try host.append(allocator, '\n');
const open_start = host.items.len;
try host.appendSlice(allocator, open_tok);
try host.append(allocator, '\n');
const content_start = host.items.len;
try host.appendSlice(allocator, seed);
const content_end = host.items.len;
try host.appendSlice(allocator, close_tok);
try host.append(allocator, '\n');
const close_end = host.items.len;
return .{ .host = try host.toOwnedSlice(allocator), .region = .{
.open_fence = Span.init(open_start, content_start),
.content = Span.init(content_start, content_end),
.close_fence = Span.init(content_end, close_end),
.body = Span.init(0, source.len),
} };
}
// .start: [ open\n ][ seed ][ close\n ][ body ]
try host.appendSlice(allocator, open_tok);
try host.append(allocator, '\n');
const content_start = host.items.len;
try host.appendSlice(allocator, seed);
const content_end = host.items.len;
try host.appendSlice(allocator, close_tok);
try host.append(allocator, '\n');
const close_end = host.items.len;
const body_start = host.items.len;
try host.appendSlice(allocator, source);
return .{ .host = try host.toOwnedSlice(allocator), .region = .{
.open_fence = Span.init(0, content_start),
.content = Span.init(content_start, content_end),
.close_fence = Span.init(content_end, close_end),
.body = Span.init(body_start, body_start + source.len),
} };
}
/// Rebuild `source`'s embedded region as a DIFFERENT archetype: keep the host
/// prose (`region.body`, byte-identical) but replace the old fences and
/// content with `to`'s convention wrapped around `new_content` — the already
/// re-serialized inner document (e.g. YAML frontmatter content re-printed as
/// JSON, for `fig convert --to-embed`). `region` must be `source`'s existing
/// region of whatever archetype it was located as (`locateRegion`/`extract`);
/// this function doesn't care what that was, only where the body is. Mirrors
/// `initRegion`'s `.start`/`.end` placement, but re-housing real content
/// rather than seeding an empty document. The returned buffer is caller-owned.
pub fn retype(allocator: Allocator, source: []const u8, region: Region, to: Type, new_content: []const u8) ![]u8 {
const a = archetypeOf(to);
const open_tok = delimLiteral(a.open);
const close_tok = delimLiteral(a.close);
const body = Span.of(u8, region.body, source);
var host: std.ArrayList(u8) = .empty;
errdefer host.deinit(allocator);
if (a.location == .end) {
// [ body ][ \n? ][ open\n ][ content ][ close\n ]
try host.appendSlice(allocator, body);
if (body.len > 0 and body[body.len - 1] != '\n') try host.append(allocator, '\n');
try host.appendSlice(allocator, open_tok);
try host.append(allocator, '\n');
try host.appendSlice(allocator, new_content);
try host.appendSlice(allocator, close_tok);
try host.append(allocator, '\n');
return host.toOwnedSlice(allocator);
}
// .start: [ open\n ][ content ][ close\n ][ body ]
try host.appendSlice(allocator, open_tok);
try host.append(allocator, '\n');
try host.appendSlice(allocator, new_content);
try host.appendSlice(allocator, close_tok);
try host.append(allocator, '\n');
try host.appendSlice(allocator, body);
return host.toOwnedSlice(allocator);
}
/// Parse an explicit content span as `t`'s inner format, no host scanning. Note
/// this parses the span VERBATIM — for a codec archetype the caller must decode
/// first (`extract` does); `parseSlice` is the shared per-format parse.
pub fn parseSpan(allocator: Allocator, source: []const u8, content: Span, t: Type) !Document {
return parseSlice(allocator, Span.of(u8, content, source), archetypeOf(t).inner);
}
/// Parse a raw content slice as `inner`.
fn parseSlice(allocator: Allocator, slice: []const u8, inner: InnerFormat) !Document {
return switch (inner) {
.yaml => if (comptime build_options.lang_yaml) blk: {
var parser = Language.YAML.Parser{ .allocator = allocator };
break :blk Language.YAML.parse(&parser, slice, Language.YAML.default_type);
} else error.FormatDisabled,
.json => if (comptime build_options.lang_json) blk: {
var parser = Language.JSON.Parser{ .allocator = allocator };
break :blk Language.JSON.parse(&parser, slice, Language.JSON.default_type);
} else error.FormatDisabled,
.fig => if (comptime build_options.lang_fig) blk: {
var parser = Language.FIG.Parser{ .allocator = allocator };
break :blk Language.FIG.parse(&parser, slice, Language.FIG.default_type);
} else error.FormatDisabled,
.toml => if (comptime build_options.lang_toml) blk: {
var parser = Language.TOML.Parser{ .allocator = allocator };
break :blk Language.TOML.parse(&parser, slice, Language.TOML.default_type);
} else error.FormatDisabled,
};
}
// --- multi-document YAML stream splitter ---------------------------------
/// Split a YAML stream into its constituent documents and parse each one
/// independently with the single-document parser.
///
/// A stream is a sequence of documents delimited by `---` (start) and `...`
/// (end) markers at column 0. Rather than teach the core parser to span
/// documents — which would complicate in-place editing and round-tripping — we
/// locate each document's byte range here (the same "locate region, then parse
/// it" shape as `extract`) and hand the slices to the parser one at a time.
///
/// Always returns at least one document: an empty or comment-only stream yields
/// a single `null` document. A parse error in any document propagates (and any
/// already-parsed documents are freed).
pub fn extractStream(allocator: Allocator, source: []const u8) !Stream {
var docs: std.ArrayList(StreamDoc) = .empty;
errdefer {
for (docs.items) |d| d.document.deinit(allocator);
docs.deinit(allocator);
}
var start: usize = 0;
if (std.mem.startsWith(u8, source, "\xEF\xBB\xBF")) start += 3; // UTF-8 BOM
var seg_start: usize = start;
var seg_explicit = false;
var line = start;
while (line < source.len) {
const next = lineEnd(source, line);
switch (markerKind(source, line)) {
.start => {
// A `---` ends the current segment and opens a new explicit
// document *at* this line — the marker stays in the slice so
// inline content (`--- foo`) is preserved and the parser eats
// the marker token.
//
// Directives (`%YAML`/`%TAG`) preceding the `---` belong to the
// document it introduces. If the pending segment is just those
// directives (and trivia), don't close it here — let the explicit
// document include them, so the parser sees `%YAML\n---\n…` as one
// document and validates the directive against its marker.
if (!segmentIsDirectives(source[seg_start..line])) {
try pushSegment(allocator, source, &docs, seg_start, line, seg_explicit);
seg_start = line;
}
seg_explicit = true;
},
.end => {
// A `...` ends the current document; the marker itself is left
// out of the slice. What follows is a fresh bare document.
try pushSegment(allocator, source, &docs, seg_start, line, seg_explicit);
seg_start = next;
seg_explicit = false;
},
.none => {},
}
line = next;
}
try pushSegment(allocator, source, &docs, seg_start, source.len, seg_explicit);
if (docs.items.len == 0) {
// An empty / trivia-only stream is one null document.
const doc = try parseYamlSlice(allocator, source[start..]);
try docs.append(allocator, .{ .content = Span.init(start, source.len), .explicit = false, .document = doc });
}
return .{ .source = source, .documents = try docs.toOwnedSlice(allocator) };
}
/// Parse `source[seg_start..seg_end]` and append it as a document, unless it is
/// a bare segment with no real content (blank/comment-only) — such a segment is
/// not a document (e.g. a leading comment before the first `---`). An explicit
/// segment (opened by `---`) is always a document, even when its body is empty.
fn pushSegment(
allocator: Allocator,
source: []const u8,
docs: *std.ArrayList(StreamDoc),
seg_start: usize,
seg_end: usize,
explicit: bool,
) !void {
const slice = source[seg_start..seg_end];
if (!explicit and !hasContent(slice)) return;
const doc = try parseYamlSlice(allocator, slice);
try docs.append(allocator, .{
.content = Span.init(seg_start, seg_end),
.explicit = explicit,
.document = doc,
});
}
fn parseYamlSlice(allocator: Allocator, slice: []const u8) !Document {
if (comptime build_options.lang_yaml) {
var parser = Language.YAML.Parser{ .allocator = allocator };
return Language.YAML.parse(&parser, slice, Language.YAML.default_type);
} else return error.FormatDisabled;
}
const MarkerKind = enum { start, end, none };
/// Classify the column-0 line at `at` as a document `---`/`...` marker. A `---`
/// marker is the bare token or a `--- `/`---\t` prefix (inline content allowed);
/// `---foo` is a plain scalar, not a marker. A `...` marker must occupy the
/// whole line. Indented lines never match (callers only pass line starts).
fn markerKind(source: []const u8, at: usize) MarkerKind {
const eol = lineEnd(source, at);
const line = std.mem.trimEnd(u8, source[at..eol], "\r\n");
if (std.mem.eql(u8, line, "---")) return .start;
if (std.mem.startsWith(u8, line, "--- ") or std.mem.startsWith(u8, line, "---\t")) return .start;
if (std.mem.eql(u8, std.mem.trimEnd(u8, line, " \t"), "...")) return .end;
return .none;
}
/// True if every content line of `slice` (ignoring blanks and comments) is a
/// column-0 directive (`%…`), and there is at least one. Such a pre-`---`
/// segment is a directives prefix that belongs to the following document, not a
/// document of its own.
fn segmentIsDirectives(slice: []const u8) bool {
var any = false;
var i: usize = 0;
while (i < slice.len) {
const eol = lineEnd(slice, i);
const line = std.mem.trimEnd(u8, slice[i..eol], "\r\n");
i = eol;
if (line.len == 0) continue;
if (line[0] == '%') {
any = true; // a directive sits at column 0
continue;
}
const body = std.mem.trim(u8, line, " \t");
if (body.len == 0 or body[0] == '#') continue; // blank or comment
return false; // some other content — not a pure directives prefix
}
return any;
}
/// True if `slice` has any line that is neither blank nor a comment.
fn hasContent(slice: []const u8) bool {
var i: usize = 0;
while (i < slice.len) {
const eol = lineEnd(slice, i);
const trimmed = std.mem.trim(u8, slice[i..eol], " \t\r\n");
if (trimmed.len != 0 and trimmed[0] != '#') return true;
i = eol;
}
return false;
}
// --- markdown frontmatter locator ---------------------------------------
fn locate(source: []const u8, a: Archetype) Error!Region {
var i: usize = 0;
if (std.mem.startsWith(u8, source, "\xEF\xBB\xBF")) i += 3; // UTF-8 BOM
const open = if (a.location == .start)
matchDelim(source, i, a.open) orelse return Error.NotFound
else
scanForDelim(source, i, a.open) orelse return Error.NotFound;
var line = open.end;
while (line < source.len) {
if (matchDelim(source, line, a.close)) |close| {
// The body is the host text outside the fences: the prefix before the
// open fence for endmatter (where the config trails the prose), else
// the suffix after the close fence — for frontmatter (`.start`) AND
// for a mid-document block (`.middle`, e.g. an HTML `<script>` data
// island). A `.middle` block also has host text BEFORE it, which this
// single-span body doesn't capture; in-place edits splice via the
// content spans and stay byte-identical on both sides regardless, so
// only `replace_body` is one-sided (it swaps the suffix).
const body = if (a.location == .end)
Span.init(0, open.start)
else
Span.init(close.end, source.len);
return .{ .open_fence = open, .content = Span.init(open.end, close.start), .close_fence = close, .body = body };
}
line = lineEnd(source, line);
}
return Error.Unterminated;
}
fn lineEnd(source: []const u8, from: usize) usize {
return if (std.mem.findScalarPos(u8, source, from, '\n')) |nl| nl + 1 else source.len;
}
/// One line vs a Delimiter; returns the line's span (incl. newline) or null.
fn matchDelim(source: []const u8, start: usize, d: Delimiter) ?Span {
const eol = lineEnd(source, start);
const line = std.mem.trimEnd(u8, source[start..eol], "\r\n");
const matched = switch (d.match) {
.whole_line => blk: {
const trimmed = std.mem.trimEnd(u8, line, " \t");
for (d.tokens) |tok| if (std.mem.eql(u8, trimmed, tok)) break :blk true;
break :blk false;
},
.prefix => blk: {
for (d.tokens) |tok| if (std.mem.startsWith(u8, line, tok)) break :blk true;
break :blk false;
},
.line_trimmed => blk: {
const trimmed = std.mem.trim(u8, line, " \t");
for (d.tokens) |tok| if (std.mem.eql(u8, trimmed, tok)) break :blk true;
break :blk false;
},
// Parametric opens: parse the line's tag and require it to resolve to the
// exact format this delimiter expects.
.fenced_open => |f| parseFencedOpen(std.mem.trim(u8, line, " \t")) == f,
.frontmatter_open => |f| parseFrontmatterOpen(std.mem.trim(u8, line, " \t")) == f,
.script_open => |f| parseScriptOpen(std.mem.trim(u8, line, " \t")) == f,
.code_open => |f| parseCodeOpen(std.mem.trim(u8, line, " \t")) == f,
};
return if (matched) Span.init(start, eol) else null;
}
fn isHspace(c: u8) bool {
return c == ' ' or c == '\t';
}
/// The value of attribute `name` in an HTML open tag's attribute region (the tag
/// name and the closing `>` stripped), or null if absent. Tolerant of attribute
/// order, extra attributes, single/double/unquoted values, and whitespace around
/// `=`; the attribute name compares case-insensitively (HTML folds them). The
/// value is returned verbatim for the caller to resolve.
fn scriptAttrValue(attrs: []const u8, name: []const u8) ?[]const u8 {
var i: usize = 0;
while (i < attrs.len) {
while (i < attrs.len and isHspace(attrs[i])) i += 1;
if (i >= attrs.len) break;
const name_start = i;
while (i < attrs.len and attrs[i] != '=' and !isHspace(attrs[i])) i += 1;
const attr_name = attrs[name_start..i];
while (i < attrs.len and isHspace(attrs[i])) i += 1;
var value: []const u8 = "";
if (i < attrs.len and attrs[i] == '=') {
i += 1;
while (i < attrs.len and isHspace(attrs[i])) i += 1;
if (i < attrs.len and (attrs[i] == '"' or attrs[i] == '\'')) {
const q = attrs[i];
i += 1;
const v_start = i;
while (i < attrs.len and attrs[i] != q) i += 1;
value = attrs[v_start..i];
if (i < attrs.len) i += 1; // consume closing quote
} else {
const v_start = i;
while (i < attrs.len and !isHspace(attrs[i])) i += 1;
value = attrs[v_start..i];
}
}
if (std.ascii.eqlIgnoreCase(attr_name, name)) return value;
}
return null;
}
/// Scan forward line-by-line for the first line matching `d`; null at EOF.
fn scanForDelim(source: []const u8, start: usize, d: Delimiter) ?Span {
var line = start;
while (line < source.len) {
if (matchDelim(source, line, d)) |span| return span;
line = lineEnd(source, line);
}
return null;
}
// --- tests ---------------------------------------------------------------
const testing = std.testing;
const AST = @import("ast/ast.zig");
fn rootKind(doc: Document) AST.Node.Kind {
return doc.ast.nodes[doc.ast.root].kind;
}
test "extract: YAML frontmatter comments survive into the parsed AST" {
if (comptime !build_options.lang_yaml) return error.SkipZigTest;
// The embed layer slices the source and hands it to the real YAML parser
// (no AST rebuild), so captured comments must ride through on `node_comments`.
const src =
\\---
\\# the title
\\title: hi # inline
\\---
\\# body
\\
;
const embedded = try extract(testing.allocator, src, .{ .frontmatter = .yaml });
defer embedded.deinit(testing.allocator);
const ast = embedded.document.ast;
try testing.expect(ast.node_comments.len > 0);
const kv = ast.nodes[ast.nodes[ast.root].kind.mapping.?].kind.keyvalue;
try testing.expectEqualStrings("the title", ast.comments(kv.key).leading[0].text);
try testing.expectEqualStrings("inline", ast.comments(kv.value).trailing.?.text);
}
test "extract: fig frontmatter (```fig fenced block) locates and parses" {
if (comptime !build_options.lang_fig) return error.SkipZigTest;
const src =
\\```fig
\\# the title
\\title = hi # inline
\\tags = [a, b]
\\```
\\# body
\\
;
const embedded = try extract(testing.allocator, src, .{ .fenced = .fig });
defer embedded.deinit(testing.allocator);
try testing.expectEqualStrings("```fig\n", src[embedded.region.open_fence.start..embedded.region.open_fence.end]);
try testing.expectEqualStrings("```\n", src[embedded.region.close_fence.start..embedded.region.close_fence.end]);
try testing.expectEqualStrings("# body\n", src[embedded.region.body.start..embedded.region.body.end]);
const ast = embedded.document.ast;
try testing.expect(ast.node_comments.len > 0);
const kv = ast.nodes[ast.nodes[ast.root].kind.mapping.?].kind.keyvalue;
try testing.expectEqualStrings("the title", ast.comments(kv.key).leading[0].text);
try testing.expectEqualStrings("inline", ast.comments(kv.value).trailing.?.text);
try testing.expectEqualSlices(u8, "hi", (try embedded.document.ast.getValByPath(&.{.{ .key = "title" }})).kind.string);
}
test "extract: a generic ```something fence is not mistaken for ```fig" {
if (comptime !build_options.lang_fig) return error.SkipZigTest;
const src =
\\```figure
\\not fig frontmatter
\\```
\\
;
try testing.expectError(Error.NotFound, locateRegion(src, .{ .fenced = .fig }));
}
test "innerFormat reports each archetype's content format" {
try testing.expectEqual(InnerFormat.yaml, innerFormat(.{ .frontmatter = .yaml }));
try testing.expectEqual(InnerFormat.yaml, innerFormat(.endmatter_yaml));
try testing.expectEqual(InnerFormat.json, innerFormat(.semicolons_json));
try testing.expectEqual(InnerFormat.fig, innerFormat(.{ .fenced = .fig }));
try testing.expectEqual(InnerFormat.toml, innerFormat(.plus_toml));
try testing.expectEqual(InnerFormat.yaml, innerFormat(.{ .fenced = .yaml }));
try testing.expectEqual(InnerFormat.json, innerFormat(.{ .fenced = .json }));
try testing.expectEqual(InnerFormat.toml, innerFormat(.{ .fenced = .toml }));
}
test "extract: TOML frontmatter (+++ fences) locates and parses" {
if (comptime !build_options.lang_toml) return error.SkipZigTest;
const src =
\\+++
\\title = "hi"
\\tags = ["a", "b"]
\\+++
\\# body
\\
;
const embedded = try extract(testing.allocator, src, .plus_toml);
defer embedded.deinit(testing.allocator);
try testing.expectEqualStrings("+++\n", src[embedded.region.open_fence.start..embedded.region.open_fence.end]);
try testing.expectEqualStrings("+++\n", src[embedded.region.close_fence.start..embedded.region.close_fence.end]);
try testing.expectEqualStrings("# body\n", src[embedded.region.body.start..embedded.region.body.end]);
try testing.expectEqualSlices(u8, "hi", (try embedded.document.ast.getValByPath(&.{.{ .key = "title" }})).kind.string);
}
test "extract: fenced ```toml / ```yaml / ```json frontmatter locate and parse" {
if (comptime !build_options.lang_toml or !build_options.lang_yaml or !build_options.lang_json)
return error.SkipZigTest;
const toml_src =
\\```toml
\\title = "hi"
\\```
\\body
\\
;
const t = try extract(testing.allocator, toml_src, .{ .fenced = .toml });
defer t.deinit(testing.allocator);
try testing.expectEqualStrings("```toml\n", toml_src[t.region.open_fence.start..t.region.open_fence.end]);
try testing.expectEqualSlices(u8, "hi", (try t.document.ast.getValByPath(&.{.{ .key = "title" }})).kind.string);
const yaml_src =
\\```yaml
\\title: hi
\\```
\\body
\\
;
const y = try extract(testing.allocator, yaml_src, .{ .fenced = .yaml });
defer y.deinit(testing.allocator);
try testing.expectEqualSlices(u8, "hi", (try y.document.ast.getValByPath(&.{.{ .key = "title" }})).kind.string);
const json_src =
\\```json
\\{"title": "hi"}
\\```
\\body
\\
;
const j = try extract(testing.allocator, json_src, .{ .fenced = .json });
defer j.deinit(testing.allocator);
try testing.expectEqualSlices(u8, "hi", (try j.document.ast.getValByPath(&.{.{ .key = "title" }})).kind.string);
}
test "extract: HTML <script type=\"application/figl\"> data island locates and parses" {
if (comptime !build_options.lang_fig) return error.SkipZigTest;
const src =
\\<!doctype html>
\\<html>
\\ <head>
\\ <script type="application/figl">
\\title = hi
\\tags = [a, b]
\\ </script>
\\ </head>
\\ <body>page</body>
\\</html>
\\
;
const embedded = try extract(testing.allocator, src, .{ .html_script = .fig });
defer embedded.deinit(testing.allocator);
// The open/close fence spans cover their whole (indented) lines; content is
// exactly what sits between them, spliced back byte-for-byte on edit.
try testing.expectEqualStrings(" <script type=\"application/figl\">\n", src[embedded.region.open_fence.start..embedded.region.open_fence.end]);
try testing.expectEqualStrings(" </script>\n", src[embedded.region.close_fence.start..embedded.region.close_fence.end]);
try testing.expectEqualStrings("title = hi\ntags = [a, b]\n", src[embedded.region.content.start..embedded.region.content.end]);
try testing.expectEqualSlices(u8, "hi", (try embedded.document.ast.getValByPath(&.{.{ .key = "title" }})).kind.string);
}
test "parseScriptOpen: tolerates quoting/order/extra attrs, resolves the format, rejects near-misses" {
// Accepted variants → the resolved format (figl ⇒ .fig).
try testing.expectEqual(InnerFormat.fig, parseScriptOpen("<script type=\"application/figl\">"));
try testing.expectEqual(InnerFormat.fig, parseScriptOpen("<script type='application/figl'>"));
try testing.expectEqual(InnerFormat.fig, parseScriptOpen("<script id=\"cfg\" type=\"application/figl\">"));
try testing.expectEqual(InnerFormat.fig, parseScriptOpen("<script type = \"application/figl\" defer>"));
try testing.expectEqual(InnerFormat.fig, parseScriptOpen("<SCRIPT TYPE=\"application/figl\">")); // HTML folds case
// Other config MIMEs resolve to their formats.
try testing.expectEqual(InnerFormat.toml, parseScriptOpen("<script type=\"application/toml\">"));
try testing.expectEqual(InnerFormat.json, parseScriptOpen("<script type=\"application/ld+json\">"));
// Rejected (null): wrong element, unknown/absent type, a same-line block.
try testing.expectEqual(@as(?InnerFormat, null), parseScriptOpen("<scripts type=\"application/figl\">"));
try testing.expectEqual(@as(?InnerFormat, null), parseScriptOpen("<script type=\"text/javascript\">"));
try testing.expectEqual(@as(?InnerFormat, null), parseScriptOpen("<script>"));
try testing.expectEqual(@as(?InnerFormat, null), parseScriptOpen("<script type=\"application/figl\">k = 1</script>"));
}
test "parseCodeOpen: matches language-/lang- tokens and the <pre> wrapper; rejects the rest" {
try testing.expectEqual(InnerFormat.fig, parseCodeOpen("<code class=\"language-figl\">"));
try testing.expectEqual(InnerFormat.fig, parseCodeOpen("<pre><code class=\"language-figl\">")); // pre wrapper
try testing.expectEqual(InnerFormat.fig, parseCodeOpen("<code class=\"lang-figl\">")); // lang- prefix
try testing.expectEqual(InnerFormat.fig, parseCodeOpen("<code class=\"hljs language-figl\">")); // token in a list
try testing.expectEqual(InnerFormat.toml, parseCodeOpen("<pre><code class='language-toml'>"));
// Rejected: no class, a non-config language, a same-line block.
try testing.expectEqual(@as(?InnerFormat, null), parseCodeOpen("<code>"));
try testing.expectEqual(@as(?InnerFormat, null), parseCodeOpen("<code class=\"language-python\">"));
try testing.expectEqual(@as(?InnerFormat, null), parseCodeOpen("<code class=\"language-figl\">k = 1</code>"));
}
test "html entity codec: decode/encode round-trip and provenance" {
// Decode maps entities back to their chars and records source provenance.
const dec = try decodeEntities(testing.allocator, "a < b & c = d");
defer dec.deinit(testing.allocator);
try testing.expectEqualStrings("a < b & c = d", dec.text);
// A decoded offset lifts back to the right source offset through the map:
// the '<' (decoded index 2) came from `<` at source index 2.
try testing.expectEqual(@as(usize, 2), dec.srcAt(2));
// Encode only touches the three significant characters.
const enc = try encodeEntities(testing.allocator, "x < y & z > w");
defer testing.allocator.free(enc);
try testing.expectEqualStrings("x < y & z > w", enc);
}
test "reencodeEdited: span-aware — untouched encoding is byte-preserved, only the edit re-encodes" {
// Original content mixes encodings: `<` (numeric) and a would-be `<`.
const orig = "title = \"a < b\"\nkeep = \"x < y\"\n";
const dec = try decodeEntities(testing.allocator, orig);
defer dec.deinit(testing.allocator);
try testing.expectEqualStrings("title = \"a < b\"\nkeep = \"x < y\"\n", dec.text);
// Simulate an editor that changed ONLY the `title` line's value to `p > q`.
const edited = "title = \"p > q\"\nkeep = \"x < y\"\n";
const out = try reencodeEdited(testing.allocator, .html_entities, orig, dec, edited);
defer testing.allocator.free(out);
// The edited value is canonically encoded (`>`→`>`); the untouched `keep`
// line keeps its ORIGINAL `<` byte-for-byte (not normalized).
try testing.expectEqualStrings("title = \"p > q\"\nkeep = \"x < y\"\n", out);
}
test "extract: <code> block decodes entities, parses, and lifts spans to source" {
if (comptime !build_options.lang_fig) return error.SkipZigTest;
const src =
\\<p>see the config:</p>
\\<pre><code class="language-figl">
\\title = hi
\\expr = "a < b"
\\</code></pre>
\\<p>after</p>
\\
;
const embedded = try extract(testing.allocator, src, .{ .html_code = .fig });
defer embedded.deinit(testing.allocator);
// The value round-trips through entity decoding.
try testing.expectEqualStrings("a < b", (try embedded.document.ast.getValByPath(&.{.{ .key = "expr" }})).kind.string);
try testing.expectEqualStrings("hi", (try embedded.document.ast.getValByPath(&.{.{ .key = "title" }})).kind.string);
}
test "detect: recognizes a <code class=\"language-…\"> block" {
try testing.expectEqual(@as(?Type, .{ .html_code = .fig }), detect("<pre><code class=\"language-figl\">\nk = v\n</code></pre>\n"));
try testing.expectEqual(@as(?Type, .{ .html_code = .toml }), detect("<article>\n<code class=\"language-toml\">\nk = 1\n</code>\n</article>\n"));
}
test "initRegion: an HTML-script archetype seeds an empty <script> island" {
if (comptime !build_options.lang_fig) return error.SkipZigTest;
const init = try initRegion(testing.allocator, "<html></html>\n", .{ .html_script = .fig });
defer testing.allocator.free(init.host);
try testing.expectEqualStrings("<script type=\"application/figl\">\n</script>\n<html></html>\n", init.host);
try testing.expectEqual(init.region.content.start, init.region.content.end);
}
test "detect: recognizes an HTML <script> data island (scanned mid-document)" {
if (comptime !build_options.lang_fig) return error.SkipZigTest;
const src = "<html><head>\n<script type=\"application/figl\">\nk = v\n</script>\n</head></html>\n";
try testing.expectEqual(@as(?Type, .{ .html_script = .fig }), detect(src));
}
test "detect: recognizes TOML and fenced-label frontmatter" {
// Content-only sniff (no parser needed): the open delimiters are distinct.
try testing.expectEqual(@as(?Type, .plus_toml), detect("+++\ntitle = \"hi\"\n+++\nbody\n"));
try testing.expectEqual(@as(?Type, .{ .fenced = .toml }), detect("```toml\ntitle = \"hi\"\n```\nbody\n"));
try testing.expectEqual(@as(?Type, .{ .fenced = .yaml }), detect("```yaml\ntitle: hi\n```\nbody\n"));
try testing.expectEqual(@as(?Type, .{ .fenced = .json }), detect("```json\n{\"t\":1}\n```\nbody\n"));
// A ```fig fence still wins its own detection, not the new fenced labels.
try testing.expectEqual(@as(?Type, .{ .fenced = .fig }), detect("```fig\nt = 1\n```\nbody\n"));
}
test "initRegion: a TOML-inner archetype seeds an empty +++ block" {
const init = try initRegion(testing.allocator, "body text\n", .plus_toml);
defer testing.allocator.free(init.host);
try testing.expectEqualStrings("+++\n+++\nbody text\n", init.host);
try testing.expectEqual(init.region.content.start, init.region.content.end);
try testing.expectEqualStrings("body text\n", init.host[init.region.body.start..init.region.body.end]);
}
test "initRegion: a fig-inner archetype seeds an empty block from nothing" {
// An empty fig document is a valid empty map (see `fig/parser.zig`), so a
// brand-new ```fig``` frontmatter block seeds empty (like YAML) and a
// subsequent set/insert lands its first key.
const init = try initRegion(testing.allocator, "body text\n", .{ .fenced = .fig });
defer testing.allocator.free(init.host);
try testing.expectEqualStrings("```fig\n```\nbody text\n", init.host);
// The seeded content is an empty span between the fences.
try testing.expectEqual(init.region.content.start, init.region.content.end);
try testing.expectEqualStrings("body text\n", init.host[init.region.body.start..init.region.body.end]);
}
test "extract: fig frontmatter with no host body" {
if (comptime !build_options.lang_fig) return error.SkipZigTest;
const src =
\\```fig
\\title = hi
\\```
\\
;
const embedded = try extract(testing.allocator, src, .{ .fenced = .fig });
defer embedded.deinit(testing.allocator);
try testing.expectEqualStrings("", src[embedded.region.body.start..embedded.region.body.end]);
}
test "extractStream: two explicit documents in a stream (JHB9)" {
if (comptime !build_options.lang_yaml) return error.SkipZigTest;
const src =
\\# Ranking of 1998 home runs
\\---
\\- Mark McGwire
\\- Sammy Sosa
\\
\\# Team ranking
\\---
\\- Chicago Cubs
\\- St Louis Cardinals
\\
;
const stream = try extractStream(testing.allocator, src);
defer stream.deinit(testing.allocator);
// The leading comment-only segment is not a document.
try testing.expectEqual(@as(usize, 2), stream.documents.len);
try testing.expect(stream.documents[0].explicit);
try testing.expect(stream.documents[1].explicit);
try testing.expectEqualSlices(u8, "Mark McGwire", (try stream.documents[0].document.ast.getValByPath(&.{.{ .index = 0 }})).kind.string);
try testing.expectEqualSlices(u8, "St Louis Cardinals", (try stream.documents[1].document.ast.getValByPath(&.{.{ .index = 1 }})).kind.string);
}
test "extractStream: directives fold into the following document (6ZKB-shaped)" {
if (comptime !build_options.lang_yaml) return error.SkipZigTest;
// Each `%…` prefix belongs to the `---` it introduces, not a document of its
// own: `Document` is doc 1, the empty `---` is doc 2, and `%YAML 1.2\n---\n…`
// is doc 3.
const src =
\\Document
\\---
\\# Empty
\\...
\\%YAML 1.2
\\---
\\matches %: 20
\\
;
const stream = try extractStream(testing.allocator, src);
defer stream.deinit(testing.allocator);
try testing.expectEqual(@as(usize, 3), stream.documents.len);
try testing.expectEqualSlices(u8, "Document", rootKind(stream.documents[0].document).string);
try testing.expect(rootKind(stream.documents[1].document) == .null_);
try testing.expectEqualSlices(u8, "20", (try stream.documents[2].document.ast.getValByPath(&.{.{ .key = "matches %" }})).kind.number.raw);
}
test "extractStream: a tag handle scoped to the first document fails later use (QLJ7)" {
if (comptime !build_options.lang_yaml) return error.SkipZigTest;
// `!prefix!` is declared only for the first document; documents 2 and 3 use
// it undeclared, so the splitter must reject the stream.
const src =
\\%TAG !prefix! tag:example.com,2011:
\\--- !prefix!A
\\a: b
\\--- !prefix!B
\\c: d
\\
;
try testing.expectError(error.UndefinedTagHandle, extractStream(testing.allocator, src));
}
test "extractStream: two document start markers yields two null docs (6XDY)" {
if (comptime !build_options.lang_yaml) return error.SkipZigTest;
const stream = try extractStream(testing.allocator, "---\n---\n");
defer stream.deinit(testing.allocator);
try testing.expectEqual(@as(usize, 2), stream.documents.len);
try testing.expect(rootKind(stream.documents[0].document) == .null_);
try testing.expect(rootKind(stream.documents[1].document) == .null_);
}
test "extractStream: document start on last line (PUW8)" {
if (comptime !build_options.lang_yaml) return error.SkipZigTest;
const stream = try extractStream(testing.allocator, "---\na: b\n---\n");
defer stream.deinit(testing.allocator);
try testing.expectEqual(@as(usize, 2), stream.documents.len);
try testing.expectEqualSlices(u8, "b", (try stream.documents[0].document.ast.getValByPath(&.{.{ .key = "a" }})).kind.string);
try testing.expect(rootKind(stream.documents[1].document) == .null_);
}
test "extractStream: bare docs separated by ... with a comment-only segment (M7A3)" {
if (comptime !build_options.lang_yaml) return error.SkipZigTest;
const src = "Bare\ndocument\n...\n# No document\n...\n|\n %!PS-Adobe-2.0 # Not the first line\n";
const stream = try extractStream(testing.allocator, src);
defer stream.deinit(testing.allocator);
// The `# No document` segment is comment-only and produces no document.
try testing.expectEqual(@as(usize, 2), stream.documents.len);
try testing.expect(!stream.documents[0].explicit);
try testing.expectEqualSlices(u8, "Bare document", rootKind(stream.documents[0].document).string);
}
test "extractStream: inline content on the marker line (L383)" {
if (comptime !build_options.lang_yaml) return error.SkipZigTest;
const stream = try extractStream(testing.allocator, "--- foo # comment\n--- foo # comment\n");
defer stream.deinit(testing.allocator);
try testing.expectEqual(@as(usize, 2), stream.documents.len);
try testing.expectEqualSlices(u8, "foo", rootKind(stream.documents[0].document).string);
try testing.expectEqualSlices(u8, "foo", rootKind(stream.documents[1].document).string);
}
test "extractStream: explicit doc then bare doc after ... (7Z25)" {
if (comptime !build_options.lang_yaml) return error.SkipZigTest;
const stream = try extractStream(testing.allocator, "---\nscalar1\n...\nkey: value\n");
defer stream.deinit(testing.allocator);
try testing.expectEqual(@as(usize, 2), stream.documents.len);
try testing.expect(stream.documents[0].explicit);
try testing.expectEqualSlices(u8, "scalar1", rootKind(stream.documents[0].document).string);
try testing.expect(!stream.documents[1].explicit);
try testing.expectEqualSlices(u8, "value", (try stream.documents[1].document.ast.getValByPath(&.{.{ .key = "key" }})).kind.string);
}
test "extractStream: single bare document" {
if (comptime !build_options.lang_yaml) return error.SkipZigTest;
const stream = try extractStream(testing.allocator, "key: value\n");
defer stream.deinit(testing.allocator);
try testing.expectEqual(@as(usize, 1), stream.documents.len);
try testing.expect(!stream.documents[0].explicit);
}
test "extractStream: empty stream is one null document" {
if (comptime !build_options.lang_yaml) return error.SkipZigTest;
const stream = try extractStream(testing.allocator, "");
defer stream.deinit(testing.allocator);
try testing.expectEqual(@as(usize, 1), stream.documents.len);
try testing.expect(rootKind(stream.documents[0].document) == .null_);
}
test "extractStream: outerSpan lifts node spans into outer coordinates" {
if (comptime !build_options.lang_yaml) return error.SkipZigTest;
const src = "---\nfoo\n---\nbar\n";
const stream = try extractStream(testing.allocator, src);
defer stream.deinit(testing.allocator);
try testing.expectEqual(@as(usize, 2), stream.documents.len);
const d1 = stream.documents[1];
const node = d1.document.ast.nodes[d1.document.ast.root];
const outer = d1.outerSpan(d1.document.span(node));
try testing.expectEqualSlices(u8, "bar", src[outer.start..outer.end]);
}
// --- Embed.detect / Embed.retype ------------------------------------------
test "detect: recognizes YAML frontmatter" {
const src = "---\ntitle: hi\n---\nbody\n";
try testing.expectEqual(@as(?Type, .{ .frontmatter = .yaml }), detect(src));
}
test "detect: recognizes JSON frontmatter" {
const src = ";;;\n{\"title\":\"hi\"}\n;;;\nbody\n";
try testing.expectEqual(@as(?Type, .semicolons_json), detect(src));
}
test "detect: recognizes fig frontmatter" {
if (comptime !build_options.lang_fig) return error.SkipZigTest;
const src = "```fig\ntitle = hi\n```\nbody\n";
try testing.expectEqual(@as(?Type, .{ .fenced = .fig }), detect(src));
}
test "detect: recognizes YAML endmatter" {
const src = "body prose\n```endmatter\ntitle: hi\n```\n";
try testing.expectEqual(@as(?Type, .endmatter_yaml), detect(src));
}
test "detect: an unterminated block is still recognized (caller sees Unterminated, not NotFound)" {
const src = "---\ntitle: hi\n";
try testing.expectEqual(@as(?Type, .{ .frontmatter = .yaml }), detect(src));
try testing.expectError(Error.Unterminated, locateRegion(src, detect(src).?));
}
test "detect: plain prose with no fences at all detects nothing" {
try testing.expectEqual(@as(?Type, null), detect("just some markdown\n\nno frontmatter here\n"));
}
test "retype: YAML frontmatter -> JSON frontmatter, body preserved byte-identical" {
const src = "---\ntitle: hi\n---\n# body\n";
const region = try locateRegion(src, .{ .frontmatter = .yaml });
const out = try retype(testing.allocator, src, region, .semicolons_json, "{\"title\":\"hi\"}\n");
defer testing.allocator.free(out);
try testing.expectEqualStrings(";;;\n{\"title\":\"hi\"}\n;;;\n# body\n", out);
}
test "retype: frontmatter -> endmatter moves the fences to the end, body first" {
const src = "---\ntitle: hi\n---\n# body\n";
const region = try locateRegion(src, .{ .frontmatter = .yaml });
const out = try retype(testing.allocator, src, region, .endmatter_yaml, "title: hi\n");
defer testing.allocator.free(out);
try testing.expectEqualStrings("# body\n```endmatter\ntitle: hi\n```\n", out);
}