//! Embedded config extraction: pull a config document out of a host file
//! (e.g. YAML frontmatter inside markdown) and parse it correctly.
const std = @import("std");
const Allocator = std.mem.Allocator;
const Language = @import("languages/language.zig");
const Document = @import("document.zig");
const Span = @import("util/span.zig");
const build_options = @import("build_options");
const Embed = @This();
const Delimiter = struct {
/// The exact line bodies this delimiter matches — used only by the FIXED
/// match modes (`whole_line`/`line_trimmed`/`prefix`). The PARAMETRIC modes
/// (`fenced_open`/`frontmatter_open`/`script_open`) ignore `tokens`: they
/// parse a language tag out of the line and resolve it, carrying the expected
/// format in the mode payload instead.
tokens: []const []const u8 = &.{},
match: Match,
/// The exact text emitted for this delimiter when *synthesizing* a region
/// (`initRegion`/`retype`), when it differs from `tokens[0]`. Required for the
/// parametric opens, whose match logic reads a tag rather than a literal —
/// e.g. `fenced` emits ```` ```toml ````, `frontmatter` emits `---toml` (or a
/// bare `---` for YAML), `script` emits the full `<script type="…">` tag.
/// Null ⇒ emit `tokens[0]` verbatim.
literal: ?[]const u8 = null,
};
/// How a `Delimiter` decides whether a host line is that delimiter.
const Match = union(enum) {
/// The line, trimmed of trailing space/tab, equals a token exactly. Leading
/// whitespace is significant (a markdown fence must sit at column 0), so an
/// indented line never matches.
whole_line,
/// The line (untrimmed) starts with a token.
prefix,
/// The line, trimmed of leading AND trailing whitespace, equals a token
/// exactly — the indentation-tolerant `whole_line`, for delimiters that may
/// sit indented inside a host (e.g. a `</script>` close tag nested in a
/// `<head>`).
line_trimmed,
/// The line opens a ```` ```<lang> ```` fenced block whose `<lang>` resolves
/// (`formatFromLangTag`) to this exact format. A bare ```` ``` ```` (no tag)
/// is a close, not an open, and never matches.
fenced_open: InnerFormat,
/// The line opens a `---<lang>` markdown-frontmatter block whose `<lang>`
/// resolves to this format; a bare `---` (no tag) resolves to `.yaml`.
frontmatter_open: InnerFormat,
/// The line, trimmed both sides, is an HTML `<script …>` open tag (on its own
/// line) whose `type` attribute's MIME resolves (`formatFromScriptMime`) to
/// this format. Attribute order, quoting, and whitespace are tolerated; a
/// same-line block (`<script …>body</script>`) is deliberately not matched.
script_open: InnerFormat,
/// The line opens an HTML `<code>` block (optionally wrapped in `<pre>`) whose
/// `class` list carries a `language-<lang>` token resolving to this format.
/// See `parseCodeOpen`.
code_open: InnerFormat,
};
/// The format an archetype's content is written in. A parametric archetype
/// carries one of these as its parameter (`fenced`/`frontmatter`/`html_script`);
/// the blessed presets each pin one. Named so callers outside this file can name
/// it too — see `innerFormat`.
///
/// Derived from the format registry: its members are exactly the entries
/// carrying an `EmbedSpellings` (`languages/language.zig`'s `dialects`), in
/// registry order — json, yaml, toml, fig. That order is NOT the one the
/// hand-written enum this replaces had (yaml, json, fig, toml), which is
/// harmless precisely here and nowhere else: `InnerFormat` is internal — it has
/// no ABI value (the C `FigEmbedType` mirror carries its own frozen integers),
/// no CLI token, and no `@intFromEnum`/`@enumFromInt`/`EnumArray` reader
/// anywhere in the tree — so reordering it is a pure renumbering of values
/// nothing observes.
pub const InnerFormat = @Enum(
Language.EnumTag(inner_format_names),
.exhaustive,
inner_format_names,
&Language.enumValues(inner_format_names),
);
const inner_format_names = Language.namesOf(.embeddable);
// Membership is now true by construction, so what is left to state is the
// INTENT the registry's `embed` fields encode: which formats have an embedded
// spelling at all. Dropping one would silently delete an archetype family
// (every `--embed fenced-<lang>`/`md-<lang>` spelling of it), and adding a
// fifth is a real decision — the CLI's `--embed` names, the C `FigEmbedType`
// mirror and the `--help` prose all have to grow with it.
comptime {
if (inner_format_names.len != 4)
@compileError("the set of formats with an embedded spelling changed — `cli/args.zig`'s" ++
" `embedTypeFromName` legacy aliases, `embed_archetype_names`, and `c_api.zig`'s" ++
" `FigEmbedType` all enumerate them by hand and must grow (or shrink) with it");
for ([_][]const u8{ "json", "yaml", "toml", "fig" }) |want| {
if (!@hasField(InnerFormat, want))
@compileError("the format-registry entry '" ++ want ++ "' no longer declares an" ++
" `embed` spelling, so its embedded form has silently disappeared");
}
// `initRegion` seeds a freshly created region with `empty_doc_seed`, and it
// has nowhere to report a format that refuses to be created from scratch:
// its whole job is to synthesize a block. A format that can be embedded
// must therefore have an empty form (`null` is XML's answer, and XML has no
// embedded spelling), which is what makes the `.?` there total.
for (inner_format_names) |n| {
if (Language.entryFor(n).empty_doc_seed == null)
@compileError("the format-registry entry '" ++ n ++ "' has an embedded spelling but no" ++
" `empty_doc_seed`, so `Embed.initRegion` has nothing to seed a new region with");
}
}
/// The `EmbedSpellings` of the registry entry `f` names. Total over
/// `InnerFormat` by construction: its members ARE the entries carrying one, so
/// the `.?` cannot fire.
fn spellings(comptime f: InnerFormat) @TypeOf(Language.entryFor(@tagName(f)).embed.?) {
return comptime Language.entryFor(@tagName(f)).embed.?;
}
/// Resolve a `fenced`/`frontmatter` language tag (a code-fence info string's
/// first token, or a `---<tag>`) to a config format — or null when it is not one
/// this project understands, so a ```` ```python ```` code sample or a `---foo`
/// line is left alone rather than mistaken for embedded config. `yml`/`figl` are
/// accepted spellings of `yaml`/`fig`.
///
/// The accepted spellings are the registry's `fence_tag` plus its
/// `fence_aliases`, per entry. Tag sets are disjoint across formats, so the
/// order this walks them in is not observable.
fn formatFromLangTag(tag: []const u8) ?InnerFormat {
const eq = std.ascii.eqlIgnoreCase;
inline for (@typeInfo(InnerFormat).@"enum".fields) |field| {
const f: InnerFormat = @enumFromInt(field.value);
const s = comptime spellings(f);
if (eq(tag, s.fence_tag)) return f;
inline for (s.fence_aliases) |alias| {
if (eq(tag, alias)) return f;
}
}
return null;
}
/// Resolve an HTML `<script type>` MIME to a config format, or null. Accepts the
/// canonical `application/<lang>` plus a few established aliases (`ld+json` for
/// JSON-LD, `application/figl` for the fig authoring dialect, `x-yaml`/`text</script>` HTML data
// island — the web's typed inert "data block" (cf. JSON-LD). It sits
// MID-document (typically in `<head>`), so it is located by scanning.
.open = .{ .match = .{ .script_open = f }, .literal = scriptLiteral(f) },
.close = .{ .tokens = &.{"</script>"}, .match = .line_trimmed },
.location = .middle,
.inner = f,
},
.html_code => |f| .{
// `<pre><code class="language-<lang>">` … `</code></pre>` — a VISIBLE,
// highlighted code block that is also the authoritative config. Its
// content is HTML-entity-encoded, so it is the one archetype with a
// non-identity codec (span-aware: see `Codec`/`reencodeEdited`).
.open = .{ .match = .{ .code_open = f }, .literal = codeLiteral(f) },
.close = .{ .tokens = &.{ "</code></pre>", "</code>" }, .match = .line_trimmed },
.location = .middle,
.inner = f,
.codec = .html_entities,
},
// --- blessed presets (distinct fixed delimiters) ---
.semicolons_json => .{
.open = .{ .tokens = &.{";;;"}, .match = .whole_line },
.close = .{ .tokens = &.{";;;"}, .match = .whole_line },
.location = .start,
.inner = .json,
},
.plus_toml => .{
.open = .{ .tokens = &.{"+++"}, .match = .whole_line },
.close = .{ .tokens = &.{"+++"}, .match = .whole_line },
.location = .start,
.inner = .toml,
},
.endmatter_yaml => .{
.open = .{ .tokens = &.{"```endmatter"}, .match = .whole_line },
.close = .{ .tokens = &.{"```"}, .match = .whole_line },
.location = .end,
.inner = .yaml,
},
};
}
/// An archetypal "config embedded in a host file" pattern. Each value fixes both
/// *where* the config lives (the host's delimiter convention) and *what* inner
/// format it is.
///
/// The three PARAMETRIC families take the format as a payload — the delimiter
/// convention is fixed but reads the format from a language tag (a fence info
/// string, a `---<lang>` tag, or a `<script type>` MIME):
/// - `frontmatter` — `---<lang>` … `---`/`...` (bare `---` ⇒ YAML). The whole
/// markdown-frontmatter ecosystem plus the self-describing `---<lang>` form.
/// - `fenced` — ```` ```<lang> ```` … ```` ``` ```` labeled code block.
/// - `html_script` — `<script type="application/<lang>">` … `</script>`.
///
/// The remaining BLESSED presets are the popular, *nonparametric* conventions
/// whose delimiter is its own distinct token (a reader decodes it without a
/// tag): `;;;` (JSON), `+++` (TOML, Hugo/Zola), and the trailing ```` ```endmatter ````
/// block. Keeping only these named — and expressing everything else
/// parametrically — is what shrinks the surface while still making every
/// (container, format) pair reachable.
pub const Type = union(enum) {
/// `---<lang>` … `---`/`...`; a bare `---` (no tag) is YAML.
frontmatter: InnerFormat,
/// ```` ```<lang> ```` … ```` ``` ````.
fenced: InnerFormat,
/// `<script type="application/<lang>">` … `</script>` HTML data island.
html_script: InnerFormat,
/// `<pre><code class="language-<lang>">` … `</code></pre>` — a visible,
/// highlighted, entity-encoded code block that is also the source of truth.
html_code: InnerFormat,
/// `;;;` … `;;;` JSON frontmatter (blessed; distinct delimiter).
semicolons_json,
/// `+++` … `+++` TOML frontmatter — the Hugo/Zola convention (blessed).
plus_toml,
/// A trailing ```` ```endmatter ```` … ```` ``` ```` YAML block. For Stephen Deken.
endmatter_yaml,
};
/// A located region, in *outer-source* byte coordinates. The fence spans are
/// retained so an editor can splice a replacement into `content` while leaving
/// everything else byte-identical.
///
/// `body_before` and `body_after` are the host text on either side of the block.
/// Together with the three region spans they TILE the source exactly:
///
/// body_before ++ open_fence ++ content ++ close_fence ++ body_after == source
///
/// Every byte of the host is in exactly one span, which is what lets a rebuild
/// (`retype`) be lossless. The UTF-8 BOM, when present, is the leading part of
/// `body_before` rather than a hole in the partition — see `regionTilesSource`,
/// which asserts the invariant, and `splitBom`, for the rebuilds that must keep
/// the BOM at offset 0.
///
/// `body` is the historical ONE-SIDED view of the same thing: the suffix after
/// the close fence for a `.start` (frontmatter) region, the prefix before the
/// open fence for an `.end` (endmatter) one. It is the read-side twin of the
/// `content` slice and the target of `replace_body`, so it names the *one* side
/// those single-slot APIs can address. For a `.middle` block (an HTML `<script>`
/// data island) that is only ever half the host, which is exactly why the two
/// explicit sides exist: prefer them for anything that reassembles the file.
pub const Region = struct {
open_fence: Span,
content: Span,
close_fence: Span,
body: Span,
/// `[0, open_fence.start)` — the host text before the block, BOM included.
body_before: Span,
/// `[close_fence.end, source.len)` — the host text after the block.
body_after: Span,
};
/// The source's leading UTF-8 BOM, if any. Named because three places need it:
/// the two locators (which must not mistake it for content) and the rebuilds,
/// which have to re-emit it at offset 0 rather than wherever `body_before`
/// would otherwise land it.
const utf8_bom = "\xEF\xBB\xBF";
/// Split `prefix` (a region's `body_before`) into its leading BOM and the real
/// host text after it. A rebuild that MOVES the block emits the BOM first and
/// the remainder wherever the host text belongs, so the BOM stays at offset 0.
fn splitBom(prefix: []const u8) struct { bom: []const u8, rest: []const u8 } {
return if (std.mem.startsWith(u8, prefix, utf8_bom))
.{ .bom = prefix[0..utf8_bom.len], .rest = prefix[utf8_bom.len..] }
else
.{ .bom = "", .rest = prefix };
}
/// Whether `region` accounts for every byte of `source` exactly once — the
/// partition invariant documented on `Region`. Asserted by the locator tests
/// across every archetype; a rebuild that starts from these spans is lossless
/// precisely when this holds.
fn regionTilesSource(region: Region, source: []const u8) bool {
return region.body_before.start == 0 and
region.body_before.end == region.open_fence.start and
region.open_fence.end == region.content.start and
region.content.end == region.close_fence.start and
region.close_fence.end == region.body_after.start and
region.body_after.end == source.len;
}
/// Extraction result. `source` is the borrowed *outer* file; `region` indexes
/// into it. `document`'s node spans are relative to the DECODED content — call
/// `outerSpan` to lift them back into outer-file coordinates (which, for a codec
/// archetype, routes through the decode provenance map). `decoded` holds the
/// content the parser actually saw: a borrow of `source` for identity archetypes,
/// an owned decoded buffer (+ map) for the entity-encoded `<code>` one.
pub const Embedded = struct {
source: []const u8,
type: Type,
region: Region,
document: Document,
decoded: Decoded,
pub fn deinit(self: Embedded, allocator: Allocator) void {
self.document.deinit(allocator);
self.decoded.deinit(allocator);
}
pub fn outerSpan(self: Embedded, s: Span) Span {
const base = self.region.content.start;
// Map decoded coordinates back to source via the provenance map (a plain
// identity for the borrow case, where `segs` is empty).
return Span.init(self.decoded.srcAt(s.start) + base, self.decoded.srcAt(s.end) + base);
}
};
/// One document within a multi-document YAML stream, located in *outer-source*
/// byte coordinates. `content` is the exact slice handed to the single-document
/// parser: it INCLUDES a leading `---` marker line (which the parser consumes)
/// but excludes a trailing `...`. `explicit` records whether the document opened
/// with a `---` marker (vs. a bare document at stream start or after `...`).
pub const StreamDoc = struct {
content: Span,
explicit: bool,
document: Document,
/// Lift a node span (relative to this document's content) into outer-file
/// coordinates — mirrors `Embedded.outerSpan`.
pub fn outerSpan(self: StreamDoc, s: Span) Span {
const base = self.content.start;
return Span.init(s.start + base, s.end + base);
}
};
/// A parsed multi-document YAML stream. `source` is the borrowed outer file;
/// each document's `content` indexes into it. The single-document parser only
/// ever sees one document at a time, so this never trips its multi-document
/// guard — the stream concept lives here, in the splitter, not in the parser.
pub const Stream = struct {
source: []const u8,
documents: []const StreamDoc,
pub fn deinit(self: Stream, allocator: Allocator) void {
for (self.documents) |d| d.document.deinit(allocator);
allocator.free(self.documents);
}
};
pub const Error = error{
/// No region of this archetype exists (plain markdown, no frontmatter).
/// Distinct from a region that exists but is malformed.
NotFound,
/// An opening delimiter with no matching close.
Unterminated,
};
/// Locate + parse the embedded document of type `t` in `source`. For a codec
/// archetype the content is decoded (with provenance) before parsing, and the
/// decoded buffer is owned by the returned `Embedded`.
pub fn extract(allocator: Allocator, source: []const u8, t: Type) !Embedded {
const a = archetypeOf(t);
const region = try locate(source, a);
const decoded = try decodeForParse(allocator, Span.of(u8, region.content, source), a.codec);
errdefer decoded.deinit(allocator);
const document = try parseSlice(allocator, decoded.text, a.inner);
return .{ .source = source, .type = t, .region = region, .document = document, .decoded = decoded };
}
/// Locate the region of type `t` in `source` without parsing its content.
/// Useful when a caller only needs the fence/content spans (e.g. to splice).
pub fn locateRegion(source: []const u8, t: Type) Error!Region {
return locate(source, archetypeOf(t));
}
/// Best-effort content sniffing for which embed archetype `source` uses — the
/// `Embed` counterpart to `Language.detect`. Recognizes an OPEN delimiter (not a
/// full `locate`, which also demands a matching close — an unterminated block
/// should still be *recognized* so the caller's `extract`/`locateRegion` surfaces
/// the real `error.Unterminated` rather than a misleading "nothing found").
///
/// The `.start` conventions are checked on the very first line (after a BOM): a
/// fenced ```` ```<lang> ````, a `---<lang>` (or bare `---`) frontmatter, or a
pub fn detect(source: []const u8) ?Type {
var i: usize = 0;
if (std.mem.startsWith(u8, source, utf8_bom)) i += utf8_bom.len;
const eol = lineEnd(source, i);
const first = std.mem.trim(u8, std.mem.trimEnd(u8, source[i..eol], "\r\n"), " \t");
if (parseFencedOpen(first)) |f| return .{ .fenced = f };
if (parseFrontmatterOpen(first)) |f| return .{ .frontmatter = f };
if (std.mem.eql(u8, first, ";;;")) return .semicolons_json;
if (std.mem.eql(u8, first, "+++")) return .plus_toml;
if (scanForDelim(source, i, archetypeOf(.endmatter_yaml).open) != null) return .endmatter_yaml;
var line = i;
while (line < source.len) : (line = lineEnd(source, line)) {
const le = lineEnd(source, line);
const lt = std.mem.trim(u8, std.mem.trimEnd(u8, source[line..le], "\r\n"), " \t");
if (parseScriptOpen(lt)) |f| return .{ .html_script = f };
if (parseCodeOpen(lt)) |f| return .{ .html_code = f };
}
return null;
}
pub fn bodyIsBefore(t: Type) bool {
return archetypeOf(t).location == .end;
}
pub fn innerFormat(t: Type) InnerFormat {
return archetypeOf(t).inner;
}
fn delimLiteral(d: Delimiter) []const u8 {
return d.literal orelse d.tokens[0];
}
pub const Initialized = struct { host: []u8, region: Region };
pub fn initRegion(allocator: Allocator, source: []const u8, t: Type) !Initialized {
const a = archetypeOf(t);
const open_tok = delimLiteral(a.open);
const close_tok = delimLiteral(a.close);
const seed: []const u8 = switch (a.inner) {
inline else => |f| comptime Language.entryFor(@tagName(f)).empty_doc_seed.?,
};
var host: std.ArrayList(u8) = .empty;
errdefer host.deinit(allocator);
if (a.location == .end) {
try host.appendSlice(allocator, source);
if (source.len > 0 and source[source.len - 1] != '\n') try host.append(allocator, '\n');
const open_start = host.items.len;
try host.appendSlice(allocator, open_tok);
try host.append(allocator, '\n');
const content_start = host.items.len;
try host.appendSlice(allocator, seed);
const content_end = host.items.len;
try host.appendSlice(allocator, close_tok);
try host.append(allocator, '\n');
const close_end = host.items.len;
const built = try host.toOwnedSlice(allocator);
const region: Region = .{
.open_fence = Span.init(open_start, content_start),
.content = Span.init(content_start, content_end),
.close_fence = Span.init(content_end, close_end),
.body = Span.init(0, open_start),
.body_before = Span.init(0, open_start),
.body_after = Span.init(close_end, close_end),
};
std.debug.assert(regionTilesSource(region, built));
return .{ .host = built, .region = region };
}
const split = splitBom(source);
try host.appendSlice(allocator, split.bom);
const open_start = host.items.len;
try host.appendSlice(allocator, open_tok);
try host.append(allocator, '\n');
const content_start = host.items.len;
try host.appendSlice(allocator, seed);
const content_end = host.items.len;
try host.appendSlice(allocator, close_tok);
try host.append(allocator, '\n');
const close_end = host.items.len;
try host.appendSlice(allocator, split.rest);
const built = try host.toOwnedSlice(allocator);
const region: Region = .{
.open_fence = Span.init(open_start, content_start),
.content = Span.init(content_start, content_end),
.close_fence = Span.init(content_end, close_end),
.body = Span.init(close_end, close_end + split.rest.len),
.body_before = Span.init(0, open_start),
.body_after = Span.init(close_end, close_end + split.rest.len),
};
std.debug.assert(regionTilesSource(region, built));
return .{ .host = built, .region = region };
}
pub const RetypeError = error{
MidDocumentRegionCannotMove,
};
pub fn retype(
allocator: Allocator,
source: []const u8,
region: Region,
from: Type,
to: Type,
new_content: []const u8,
) (RetypeError || Allocator.Error)![]u8 {
const a = archetypeOf(to);
const src_loc = archetypeOf(from).location;
if (src_loc == .middle and a.location != .middle) return RetypeError.MidDocumentRegionCannotMove;
const open_tok = delimLiteral(a.open);
const close_tok = delimLiteral(a.close);
const split = splitBom(Span.of(u8, region.body_before, source));
const before = split.rest;
const after = Span.of(u8, region.body_after, source);
var host: std.ArrayList(u8) = .empty;
errdefer host.deinit(allocator);
const block = struct {
fn emit(al: Allocator, h: *std.ArrayList(u8), o: []const u8, c: []const u8, inner: []const u8) !void {
try h.appendSlice(al, o);
try h.append(al, '\n');
try h.appendSlice(al, inner);
try h.appendSlice(al, c);
try h.append(al, '\n');
}
}.emit;
const in_place = a.location == .middle or a.location == src_loc;
try host.appendSlice(allocator, split.bom);
if (in_place) {
try host.appendSlice(allocator, before);
try block(allocator, &host, open_tok, close_tok, new_content);
try host.appendSlice(allocator, after);
} else if (a.location == .start) {
try block(allocator, &host, open_tok, close_tok, new_content);
try host.appendSlice(allocator, before);
try host.appendSlice(allocator, after);
} else {
try host.appendSlice(allocator, before);
try host.appendSlice(allocator, after);
const prose = host.items[split.bom.len..];
if (prose.len > 0 and prose[prose.len - 1] != '\n') try host.append(allocator, '\n');
try block(allocator, &host, open_tok, close_tok, new_content);
}
return host.toOwnedSlice(allocator);
}
pub fn parseSpan(allocator: Allocator, source: []const u8, content: Span, t: Type) !Document {
return parseSlice(allocator, Span.of(u8, content, source), archetypeOf(t).inner);
}
fn parseSlice(allocator: Allocator, slice: []const u8, inner: InnerFormat) !Document {
return switch (inner) {
inline else => |f| {
const d = comptime Language.entryFor(@tagName(f));
if (comptime d.Lang == void) return error.FormatDisabled;
var parser = d.Lang.Parser{ .allocator = allocator };
return d.Lang.parse(&parser, slice, d.dialect);
},
};
}
pub fn extractStream(allocator: Allocator, source: []const u8) !Stream {
var docs: std.ArrayList(StreamDoc) = .empty;
errdefer {
for (docs.items) |d| d.document.deinit(allocator);
docs.deinit(allocator);
}
var start: usize = 0;
if (std.mem.startsWith(u8, source, "\xEF\xBB\xBF")) start += 3;
var seg_start: usize = start;
var seg_explicit = false;
var line = start;
while (line < source.len) {
const next = lineEnd(source, line);
switch (markerKind(source, line)) {
.start => {
if (!segmentIsDirectives(source[seg_start..line])) {
try pushSegment(allocator, source, &docs, seg_start, line, seg_explicit);
seg_start = line;
}
seg_explicit = true;
},
.end => {
try pushSegment(allocator, source, &docs, seg_start, line, seg_explicit);
seg_start = next;
seg_explicit = false;
},
.none => {},
}
line = next;
}
try pushSegment(allocator, source, &docs, seg_start, source.len, seg_explicit);
if (docs.items.len == 0) {
const doc = try parseYamlSlice(allocator, source[start..]);
try docs.append(allocator, .{ .content = Span.init(start, source.len), .explicit = false, .document = doc });
}
return .{ .source = source, .documents = try docs.toOwnedSlice(allocator) };
}
fn pushSegment(
allocator: Allocator,
source: []const u8,
docs: *std.ArrayList(StreamDoc),
seg_start: usize,
seg_end: usize,
explicit: bool,
) !void {
const slice = source[seg_start..seg_end];
if (!explicit and !hasContent(slice)) return;
const doc = try parseYamlSlice(allocator, slice);
try docs.append(allocator, .{
.content = Span.init(seg_start, seg_end),
.explicit = explicit,
.document = doc,
});
}
fn parseYamlSlice(allocator: Allocator, slice: []const u8) !Document {
if (comptime build_options.lang_yaml) {
var parser = Language.YAML.Parser{ .allocator = allocator };
return Language.YAML.parse(&parser, slice, Language.YAML.default_type);
} else return error.FormatDisabled;
}
const MarkerKind = enum { start, end, none };
fn markerKind(source: []const u8, at: usize) MarkerKind {
const eol = lineEnd(source, at);
const line = std.mem.trimEnd(u8, source[at..eol], "\r\n");
if (std.mem.eql(u8, line, "---")) return .start;
if (std.mem.startsWith(u8, line, "--- ") or std.mem.startsWith(u8, line, "---\t")) return .start;
if (std.mem.eql(u8, std.mem.trimEnd(u8, line, " \t"), "...")) return .end;
return .none;
}
fn segmentIsDirectives(slice: []const u8) bool {
var any = false;
var i: usize = 0;
while (i < slice.len) {
const eol = lineEnd(slice, i);
const line = std.mem.trimEnd(u8, slice[i..eol], "\r\n");
i = eol;
if (line.len == 0) continue;
if (line[0] == '%') {
any = true; continue;
}
const body = std.mem.trim(u8, line, " \t");
if (body.len == 0 or body[0] == '#') continue; return false; }
return any;
}
fn hasContent(slice: []const u8) bool {
var i: usize = 0;
while (i < slice.len) {
const eol = lineEnd(slice, i);
const trimmed = std.mem.trim(u8, slice[i..eol], " \t\r\n");
if (trimmed.len != 0 and trimmed[0] != '#') return true;
i = eol;
}
return false;
}
fn locate(source: []const u8, a: Archetype) Error!Region {
var i: usize = 0;
if (std.mem.startsWith(u8, source, utf8_bom)) i += utf8_bom.len;
const open = if (a.location == .start)
matchDelim(source, i, a.open) orelse return Error.NotFound
else
scanForDelim(source, i, a.open) orelse return Error.NotFound;
var line = open.end;
while (line < source.len) {
if (matchDelim(source, line, a.close)) |close| {
const body = if (a.location == .end)
Span.init(0, open.start)
else
Span.init(close.end, source.len);
const region: Region = .{
.open_fence = open,
.content = Span.init(open.end, close.start),
.close_fence = close,
.body = body,
.body_before = Span.init(0, open.start),
.body_after = Span.init(close.end, source.len),
};
std.debug.assert(regionTilesSource(region, source));
return region;
}
line = lineEnd(source, line);
}
return Error.Unterminated;
}
fn lineEnd(source: []const u8, from: usize) usize {
return if (std.mem.findScalarPos(u8, source, from, '\n')) |nl| nl + 1 else source.len;
}
fn matchDelim(source: []const u8, start: usize, d: Delimiter) ?Span {
const eol = lineEnd(source, start);
const line = std.mem.trimEnd(u8, source[start..eol], "\r\n");
const matched = switch (d.match) {
.whole_line => blk: {
const trimmed = std.mem.trimEnd(u8, line, " \t");
for (d.tokens) |tok| if (std.mem.eql(u8, trimmed, tok)) break :blk true;
break :blk false;
},
.prefix => blk: {
for (d.tokens) |tok| if (std.mem.startsWith(u8, line, tok)) break :blk true;
break :blk false;
},
.line_trimmed => blk: {
const trimmed = std.mem.trim(u8, line, " \t");
for (d.tokens) |tok| if (std.mem.eql(u8, trimmed, tok)) break :blk true;
break :blk false;
},
.fenced_open => |f| parseFencedOpen(std.mem.trim(u8, line, " \t")) == f,
.frontmatter_open => |f| parseFrontmatterOpen(std.mem.trim(u8, line, " \t")) == f,
.script_open => |f| parseScriptOpen(std.mem.trim(u8, line, " \t")) == f,
.code_open => |f| parseCodeOpen(std.mem.trim(u8, line, " \t")) == f,
};
return if (matched) Span.init(start, eol) else null;
}
fn isHspace(c: u8) bool {
return c == ' ' or c == '\t';
}
fn scriptAttrValue(attrs: []const u8, name: []const u8) ?[]const u8 {
var i: usize = 0;
while (i < attrs.len) {
while (i < attrs.len and isHspace(attrs[i])) i += 1;
if (i >= attrs.len) break;
const name_start = i;
while (i < attrs.len and attrs[i] != '=' and !isHspace(attrs[i])) i += 1;
const attr_name = attrs[name_start..i];
while (i < attrs.len and isHspace(attrs[i])) i += 1;
var value: []const u8 = "";
if (i < attrs.len and attrs[i] == '=') {
i += 1;
while (i < attrs.len and isHspace(attrs[i])) i += 1;
if (i < attrs.len and (attrs[i] == '"' or attrs[i] == '\'')) {
const q = attrs[i];
i += 1;
const v_start = i;
while (i < attrs.len and attrs[i] != q) i += 1;
value = attrs[v_start..i];
if (i < attrs.len) i += 1; } else {
const v_start = i;
while (i < attrs.len and !isHspace(attrs[i])) i += 1;
value = attrs[v_start..i];
}
}
if (std.ascii.eqlIgnoreCase(attr_name, name)) return value;
}
return null;
}
fn scanForDelim(source: []const u8, start: usize, d: Delimiter) ?Span {
var line = start;
while (line < source.len) {
if (matchDelim(source, line, d)) |span| return span;
line = lineEnd(source, line);
}
return null;
}
const testing = std.testing;
const AST = @import("ast/ast.zig");
fn rootKind(doc: Document) AST.Node.Kind {
return doc.ast.nodes[doc.ast.root].kind;
}
test "extract: YAML frontmatter comments survive into the parsed AST" {
if (comptime !build_options.lang_yaml) return error.SkipZigTest;
const src =
\\---
\\# the title
\\title: hi # inline
\\---
\\# body
\\
;
const embedded = try extract(testing.allocator, src, .{ .frontmatter = .yaml });
defer embedded.deinit(testing.allocator);
const ast = embedded.document.ast;
try testing.expect(ast.node_comments.len > 0);
const kv = ast.nodes[ast.nodes[ast.root].kind.mapping.?].kind.keyvalue;
try testing.expectEqualStrings("the title", ast.comments(kv.key).leading[0].text);
try testing.expectEqualStrings("inline", ast.comments(kv.value).trailing.?.text);
}
test "extract: fig frontmatter (```fig fenced block) locates and parses" {
if (comptime !build_options.lang_fig) return error.SkipZigTest;
const src =
\\```fig
\\# the title
\\title = hi # inline
\\tags = [a, b]
\\```
\\# body
\\
;
const embedded = try extract(testing.allocator, src, .{ .fenced = .fig });
defer embedded.deinit(testing.allocator);
try testing.expectEqualStrings("```fig\n", src[embedded.region.open_fence.start..embedded.region.open_fence.end]);
try testing.expectEqualStrings("```\n", src[embedded.region.close_fence.start..embedded.region.close_fence.end]);
try testing.expectEqualStrings("# body\n", src[embedded.region.body.start..embedded.region.body.end]);
const ast = embedded.document.ast;
try testing.expect(ast.node_comments.len > 0);
const kv = ast.nodes[ast.nodes[ast.root].kind.mapping.?].kind.keyvalue;
try testing.expectEqualStrings("the title", ast.comments(kv.key).leading[0].text);
try testing.expectEqualStrings("inline", ast.comments(kv.value).trailing.?.text);
try testing.expectEqualSlices(u8, "hi", (try embedded.document.ast.getValByPath(&.{.{ .key = "title" }})).kind.string);
}
test "extract: a generic ```something fence is not mistaken for ```fig" {
if (comptime !build_options.lang_fig) return error.SkipZigTest;
const src =
\\```figure
\\not fig frontmatter
\\```
\\
;
try testing.expectError(Error.NotFound, locateRegion(src, .{ .fenced = .fig }));
}
test "innerFormat reports each archetype's content format" {
try testing.expectEqual(InnerFormat.yaml, innerFormat(.{ .frontmatter = .yaml }));
try testing.expectEqual(InnerFormat.yaml, innerFormat(.endmatter_yaml));
try testing.expectEqual(InnerFormat.json, innerFormat(.semicolons_json));
try testing.expectEqual(InnerFormat.fig, innerFormat(.{ .fenced = .fig }));
try testing.expectEqual(InnerFormat.toml, innerFormat(.plus_toml));
try testing.expectEqual(InnerFormat.yaml, innerFormat(.{ .fenced = .yaml }));
try testing.expectEqual(InnerFormat.json, innerFormat(.{ .fenced = .json }));
try testing.expectEqual(InnerFormat.toml, innerFormat(.{ .fenced = .toml }));
}
test "the four spelling tables are derived, and still spell exactly what they used to" {
try testing.expectEqualStrings("```yaml", fencedLiteral(.yaml));
try testing.expectEqualStrings("```json", fencedLiteral(.json));
try testing.expectEqualStrings("```toml", fencedLiteral(.toml));
try testing.expectEqualStrings("```fig", fencedLiteral(.fig));
try testing.expectEqualStrings("---", frontmatterLiteral(.yaml)); try testing.expectEqualStrings("---json", frontmatterLiteral(.json));
try testing.expectEqualStrings("---toml", frontmatterLiteral(.toml));
try testing.expectEqualStrings("---fig", frontmatterLiteral(.fig));
try testing.expectEqualStrings("<script type=\"application/yaml\">", scriptLiteral(.yaml));
try testing.expectEqualStrings("<script type=\"application/json\">", scriptLiteral(.json));
try testing.expectEqualStrings("<script type=\"application/toml\">", scriptLiteral(.toml));
try testing.expectEqualStrings("<script type=\"application/figl\">", scriptLiteral(.fig));
try testing.expectEqualStrings("<pre><code class=\"language-yaml\">", codeLiteral(.yaml));
try testing.expectEqualStrings("<pre><code class=\"language-json\">", codeLiteral(.json));
try testing.expectEqualStrings("<pre><code class=\"language-toml\">", codeLiteral(.toml));
try testing.expectEqualStrings("<pre><code class=\"language-figl\">", codeLiteral(.fig));
try testing.expectEqual(InnerFormat.yaml, formatFromLangTag("yaml"));
try testing.expectEqual(InnerFormat.yaml, formatFromLangTag("yml"));
try testing.expectEqual(InnerFormat.yaml, formatFromLangTag("YAML"));
try testing.expectEqual(InnerFormat.json, formatFromLangTag("json"));
try testing.expectEqual(InnerFormat.toml, formatFromLangTag("toml"));
try testing.expectEqual(InnerFormat.fig, formatFromLangTag("fig"));
try testing.expectEqual(InnerFormat.fig, formatFromLangTag("figl"));
try testing.expectEqual(@as(?InnerFormat, null), formatFromLangTag("python"));
try testing.expectEqual(@as(?InnerFormat, null), formatFromLangTag("endmatter"));
try testing.expectEqual(InnerFormat.yaml, formatFromScriptMime("application/yaml"));
try testing.expectEqual(InnerFormat.yaml, formatFromScriptMime("application/x-yaml"));
try testing.expectEqual(InnerFormat.yaml, formatFromScriptMime("text/yaml"));
try testing.expectEqual(InnerFormat.json, formatFromScriptMime("application/json"));
try testing.expectEqual(InnerFormat.json, formatFromScriptMime("application/ld+json"));
try testing.expectEqual(InnerFormat.toml, formatFromScriptMime("application/toml"));
try testing.expectEqual(InnerFormat.fig, formatFromScriptMime("application/figl"));
try testing.expectEqual(InnerFormat.fig, formatFromScriptMime("application/fig"));
try testing.expectEqual(@as(?InnerFormat, null), formatFromScriptMime("text/javascript"));
}
test "extract: TOML frontmatter (+++ fences) locates and parses" {
if (comptime !build_options.lang_toml) return error.SkipZigTest;
const src =
\\+++
\\title = "hi"
\\tags = ["a", "b"]
\\+++
\\# body
\\
;
const embedded = try extract(testing.allocator, src, .plus_toml);
defer embedded.deinit(testing.allocator);
try testing.expectEqualStrings("+++\n", src[embedded.region.open_fence.start..embedded.region.open_fence.end]);
try testing.expectEqualStrings("+++\n", src[embedded.region.close_fence.start..embedded.region.close_fence.end]);
try testing.expectEqualStrings("# body\n", src[embedded.region.body.start..embedded.region.body.end]);
try testing.expectEqualSlices(u8, "hi", (try embedded.document.ast.getValByPath(&.{.{ .key = "title" }})).kind.string);
}
test "extract: fenced ```toml / ```yaml / ```json frontmatter locate and parse" {
if (comptime !build_options.lang_toml or !build_options.lang_yaml or !build_options.lang_json)
return error.SkipZigTest;
const toml_src =
\\```toml
\\title = "hi"
\\```
\\body
\\
;
const t = try extract(testing.allocator, toml_src, .{ .fenced = .toml });
defer t.deinit(testing.allocator);
try testing.expectEqualStrings("```toml\n", toml_src[t.region.open_fence.start..t.region.open_fence.end]);
try testing.expectEqualSlices(u8, "hi", (try t.document.ast.getValByPath(&.{.{ .key = "title" }})).kind.string);
const yaml_src =
\\```yaml
\\title: hi
\\```
\\body
\\
;
const y = try extract(testing.allocator, yaml_src, .{ .fenced = .yaml });
defer y.deinit(testing.allocator);
try testing.expectEqualSlices(u8, "hi", (try y.document.ast.getValByPath(&.{.{ .key = "title" }})).kind.string);
const json_src =
\\```json
\\{"title": "hi"}
\\```
\\body
\\
;
const j = try extract(testing.allocator, json_src, .{ .fenced = .json });
defer j.deinit(testing.allocator);
try testing.expectEqualSlices(u8, "hi", (try j.document.ast.getValByPath(&.{.{ .key = "title" }})).kind.string);
}
test "extract: HTML <script type=\"application/figl\"> data island locates and parses" {
if (comptime !build_options.lang_fig) return error.SkipZigTest;
const src =
\\<!doctype html>
\\<html>
\\ <head>
\\ <script type="application/figl">
\\title = hi
\\tags = [a, b]
\\ </script>
\\ </head>
\\ <body>page</body>
\\</html>
\\
;
const embedded = try extract(testing.allocator, src, .{ .html_script = .fig });
defer embedded.deinit(testing.allocator);
// The open/close fence spans cover their whole (indented) lines; content is
// exactly what sits between them, spliced back byte-for-byte on edit.
try testing.expectEqualStrings(" <script type=\"application/figl\">\n", src[embedded.region.open_fence.start..embedded.region.open_fence.end]);
try testing.expectEqualStrings(" </script>\n", src[embedded.region.close_fence.start..embedded.region.close_fence.end]);
try testing.expectEqualStrings("title = hi\ntags = [a, b]\n", src[embedded.region.content.start..embedded.region.content.end]);
try testing.expectEqualSlices(u8, "hi", (try embedded.document.ast.getValByPath(&.{.{ .key = "title" }})).kind.string);
}
test "parseScriptOpen: tolerates quoting/order/extra attrs, resolves the format, rejects near-misses" {
// Accepted variants → the resolved format (figl ⇒ .fig).
try testing.expectEqual(InnerFormat.fig, parseScriptOpen("<script type=\"application/figl\">"));
try testing.expectEqual(InnerFormat.fig, parseScriptOpen("<script type='application/figl'>"));
try testing.expectEqual(InnerFormat.fig, parseScriptOpen("<script id=\"cfg\" type=\"application/figl\">"));
try testing.expectEqual(InnerFormat.fig, parseScriptOpen("<script type = \"application/figl\" defer>"));
try testing.expectEqual(InnerFormat.fig, parseScriptOpen("<SCRIPT TYPE=\"application/figl\">")); // HTML folds case
// Other config MIMEs resolve to their formats.
try testing.expectEqual(InnerFormat.toml, parseScriptOpen("<script type=\"application/toml\">"));
try testing.expectEqual(InnerFormat.json, parseScriptOpen("<script type=\"application/ld+json\">"));
// Rejected (null): wrong element, unknown/absent type, a same-line block.
try testing.expectEqual(@as(?InnerFormat, null), parseScriptOpen("<scripts type=\"application/figl\">"));
try testing.expectEqual(@as(?InnerFormat, null), parseScriptOpen("<script type=\"text/javascript\">"));
try testing.expectEqual(@as(?InnerFormat, null), parseScriptOpen("<script>"));
try testing.expectEqual(@as(?InnerFormat, null), parseScriptOpen("<script type=\"application/figl\">k = 1</script>"));
}
test "parseCodeOpen: matches language-/lang- tokens and the <pre> wrapper; rejects the rest" {
try testing.expectEqual(InnerFormat.fig, parseCodeOpen("<code class=\"language-figl\">"));
try testing.expectEqual(InnerFormat.fig, parseCodeOpen("<pre><code class=\"language-figl\">")); // pre wrapper
try testing.expectEqual(InnerFormat.fig, parseCodeOpen("<code class=\"lang-figl\">")); // lang- prefix
try testing.expectEqual(InnerFormat.fig, parseCodeOpen("<code class=\"hljs language-figl\">")); // token in a list
try testing.expectEqual(InnerFormat.toml, parseCodeOpen("<pre><code class='language-toml'>"));
// Rejected: no class, a non-config language, a same-line block.
try testing.expectEqual(@as(?InnerFormat, null), parseCodeOpen("<code>"));
try testing.expectEqual(@as(?InnerFormat, null), parseCodeOpen("<code class=\"language-python\">"));
try testing.expectEqual(@as(?InnerFormat, null), parseCodeOpen("<code class=\"language-figl\">k = 1</code>"));
}
test "html entity codec: decode/encode round-trip and provenance" {
// Decode maps entities back to their chars and records source provenance.
const dec = try decodeEntities(testing.allocator, "a < b & c = d");
defer dec.deinit(testing.allocator);
try testing.expectEqualStrings("a < b & c = d", dec.text);
// A decoded offset lifts back to the right source offset through the map:
// the '<' (decoded index 2) came from `<` at source index 2.
try testing.expectEqual(@as(usize, 2), dec.srcAt(2));
// Encode only touches the three significant characters.
const enc = try encodeEntities(testing.allocator, "x < y & z > w");
defer testing.allocator.free(enc);
try testing.expectEqualStrings("x < y & z > w", enc);
}
test "reencodeEdited: span-aware — untouched encoding is byte-preserved, only the edit re-encodes" {
// Original content mixes encodings: `<` (numeric) and a would-be `<`.
const orig = "title = \"a < b\"\nkeep = \"x < y\"\n";
const dec = try decodeEntities(testing.allocator, orig);
defer dec.deinit(testing.allocator);
try testing.expectEqualStrings("title = \"a < b\"\nkeep = \"x < y\"\n", dec.text);
// Simulate an editor that changed ONLY the `title` line's value to `p > q`.
const edited = "title = \"p > q\"\nkeep = \"x < y\"\n";
const out = try reencodeEdited(testing.allocator, .html_entities, orig, dec, edited);
defer testing.allocator.free(out);
// The edited value is canonically encoded (`>`→`>`); the untouched `keep`
// line keeps its ORIGINAL `<` byte-for-byte (not normalized).
try testing.expectEqualStrings("title = \"p > q\"\nkeep = \"x < y\"\n", out);
}
test "extract: <code> block decodes entities, parses, and lifts spans to source" {
if (comptime !build_options.lang_fig) return error.SkipZigTest;
const src =
\\<p>see the config:</p>
\\<pre><code class="language-figl">
\\title = hi
\\expr = "a < b"
\\</code></pre>
\\<p>after</p>
\\
;
const embedded = try extract(testing.allocator, src, .{ .html_code = .fig });
defer embedded.deinit(testing.allocator);
// The value round-trips through entity decoding.
try testing.expectEqualStrings("a < b", (try embedded.document.ast.getValByPath(&.{.{ .key = "expr" }})).kind.string);
try testing.expectEqualStrings("hi", (try embedded.document.ast.getValByPath(&.{.{ .key = "title" }})).kind.string);
}
test "detect: recognizes a <code class=\"language-…\"> block" {
try testing.expectEqual(@as(?Type, .{ .html_code = .fig }), detect("<pre><code class=\"language-figl\">\nk = v\n</code></pre>\n"));
try testing.expectEqual(@as(?Type, .{ .html_code = .toml }), detect("<article>\n<code class=\"language-toml\">\nk = 1\n</code>\n</article>\n"));
}
test "initRegion: an HTML-script archetype seeds an empty <script> island" {
if (comptime !build_options.lang_fig) return error.SkipZigTest;
const init = try initRegion(testing.allocator, "<html></html>\n", .{ .html_script = .fig });
defer testing.allocator.free(init.host);
try testing.expectEqualStrings("<script type=\"application/figl\">\n</script>\n<html></html>\n", init.host);
try testing.expectEqual(init.region.content.start, init.region.content.end);
}
test "detect: recognizes an HTML <script> data island (scanned mid-document)" {
if (comptime !build_options.lang_fig) return error.SkipZigTest;
const src = "<html><head>\n<script type=\"application/figl\">\nk = v\n</script>\n</head></html>\n";
try testing.expectEqual(@as(?Type, .{ .html_script = .fig }), detect(src));
}
test "detect: recognizes TOML and fenced-label frontmatter" {
// Content-only sniff (no parser needed): the open delimiters are distinct.
try testing.expectEqual(@as(?Type, .plus_toml), detect("+++\ntitle = \"hi\"\n+++\nbody\n"));
try testing.expectEqual(@as(?Type, .{ .fenced = .toml }), detect("```toml\ntitle = \"hi\"\n```\nbody\n"));
try testing.expectEqual(@as(?Type, .{ .fenced = .yaml }), detect("```yaml\ntitle: hi\n```\nbody\n"));
try testing.expectEqual(@as(?Type, .{ .fenced = .json }), detect("```json\n{\"t\":1}\n```\nbody\n"));
// A ```fig fence still wins its own detection, not the new fenced labels.
try testing.expectEqual(@as(?Type, .{ .fenced = .fig }), detect("```fig\nt = 1\n```\nbody\n"));
}
test "initRegion: a TOML-inner archetype seeds an empty +++ block" {
const init = try initRegion(testing.allocator, "body text\n", .plus_toml);
defer testing.allocator.free(init.host);
try testing.expectEqualStrings("+++\n+++\nbody text\n", init.host);
try testing.expectEqual(init.region.content.start, init.region.content.end);
try testing.expectEqualStrings("body text\n", init.host[init.region.body.start..init.region.body.end]);
}
test "initRegion: a fig-inner archetype seeds an empty block from nothing" {
// An empty fig document is a valid empty map (see `fig/parser.zig`), so a
// brand-new ```fig``` frontmatter block seeds empty (like YAML) and a
// subsequent set/insert lands its first key.
const init = try initRegion(testing.allocator, "body text\n", .{ .fenced = .fig });
defer testing.allocator.free(init.host);
try testing.expectEqualStrings("```fig\n```\nbody text\n", init.host);
// The seeded content is an empty span between the fences.
try testing.expectEqual(init.region.content.start, init.region.content.end);
try testing.expectEqualStrings("body text\n", init.host[init.region.body.start..init.region.body.end]);
}
test "extract: fig frontmatter with no host body" {
if (comptime !build_options.lang_fig) return error.SkipZigTest;
const src =
\\```fig
\\title = hi
\\```
\\
;
const embedded = try extract(testing.allocator, src, .{ .fenced = .fig });
defer embedded.deinit(testing.allocator);
try testing.expectEqualStrings("", src[embedded.region.body.start..embedded.region.body.end]);
}
test "extractStream: two explicit documents in a stream (JHB9)" {
if (comptime !build_options.lang_yaml) return error.SkipZigTest;
const src =
\\# Ranking of 1998 home runs
\\---
\\- Mark McGwire
\\- Sammy Sosa
\\
\\# Team ranking
\\---
\\- Chicago Cubs
\\- St Louis Cardinals
\\
;
const stream = try extractStream(testing.allocator, src);
defer stream.deinit(testing.allocator);
// The leading comment-only segment is not a document.
try testing.expectEqual(@as(usize, 2), stream.documents.len);
try testing.expect(stream.documents[0].explicit);
try testing.expect(stream.documents[1].explicit);
try testing.expectEqualSlices(u8, "Mark McGwire", (try stream.documents[0].document.ast.getValByPath(&.{.{ .index = 0 }})).kind.string);
try testing.expectEqualSlices(u8, "St Louis Cardinals", (try stream.documents[1].document.ast.getValByPath(&.{.{ .index = 1 }})).kind.string);
}
test "extractStream: directives fold into the following document (6ZKB-shaped)" {
if (comptime !build_options.lang_yaml) return error.SkipZigTest;
// Each `%…` prefix belongs to the `---` it introduces, not a document of its
// own: `Document` is doc 1, the empty `---` is doc 2, and `%YAML 1.2\n---\n…`
// is doc 3.
const src =
\\Document
\\---
\\# Empty
\\...
\\%YAML 1.2
\\---
\\matches %: 20
\\
;
const stream = try extractStream(testing.allocator, src);
defer stream.deinit(testing.allocator);
try testing.expectEqual(@as(usize, 3), stream.documents.len);
try testing.expectEqualSlices(u8, "Document", rootKind(stream.documents[0].document).string);
try testing.expect(rootKind(stream.documents[1].document) == .null_);
try testing.expectEqualSlices(u8, "20", (try stream.documents[2].document.ast.getValByPath(&.{.{ .key = "matches %" }})).kind.number.raw);
}
test "extractStream: a tag handle scoped to the first document fails later use (QLJ7)" {
if (comptime !build_options.lang_yaml) return error.SkipZigTest;
// `!prefix!` is declared only for the first document; documents 2 and 3 use
// it undeclared, so the splitter must reject the stream.
const src =
\\%TAG !prefix! tag:example.com,2011:
\\--- !prefix!A
\\a: b
\\--- !prefix!B
\\c: d
\\
;
try testing.expectError(error.UndefinedTagHandle, extractStream(testing.allocator, src));
}
test "extractStream: two document start markers yields two null docs (6XDY)" {
if (comptime !build_options.lang_yaml) return error.SkipZigTest;
const stream = try extractStream(testing.allocator, "---\n---\n");
defer stream.deinit(testing.allocator);
try testing.expectEqual(@as(usize, 2), stream.documents.len);
try testing.expect(rootKind(stream.documents[0].document) == .null_);
try testing.expect(rootKind(stream.documents[1].document) == .null_);
}
test "extractStream: document start on last line (PUW8)" {
if (comptime !build_options.lang_yaml) return error.SkipZigTest;
const stream = try extractStream(testing.allocator, "---\na: b\n---\n");
defer stream.deinit(testing.allocator);
try testing.expectEqual(@as(usize, 2), stream.documents.len);
try testing.expectEqualSlices(u8, "b", (try stream.documents[0].document.ast.getValByPath(&.{.{ .key = "a" }})).kind.string);
try testing.expect(rootKind(stream.documents[1].document) == .null_);
}
test "extractStream: bare docs separated by ... with a comment-only segment (M7A3)" {
if (comptime !build_options.lang_yaml) return error.SkipZigTest;
const src = "Bare\ndocument\n...\n# No document\n...\n|\n %!PS-Adobe-2.0 # Not the first line\n";
const stream = try extractStream(testing.allocator, src);
defer stream.deinit(testing.allocator);
// The `# No document` segment is comment-only and produces no document.
try testing.expectEqual(@as(usize, 2), stream.documents.len);
try testing.expect(!stream.documents[0].explicit);
try testing.expectEqualSlices(u8, "Bare document", rootKind(stream.documents[0].document).string);
}
test "extractStream: inline content on the marker line (L383)" {
if (comptime !build_options.lang_yaml) return error.SkipZigTest;
const stream = try extractStream(testing.allocator, "--- foo # comment\n--- foo # comment\n");
defer stream.deinit(testing.allocator);
try testing.expectEqual(@as(usize, 2), stream.documents.len);
try testing.expectEqualSlices(u8, "foo", rootKind(stream.documents[0].document).string);
try testing.expectEqualSlices(u8, "foo", rootKind(stream.documents[1].document).string);
}
test "extractStream: explicit doc then bare doc after ... (7Z25)" {
if (comptime !build_options.lang_yaml) return error.SkipZigTest;
const stream = try extractStream(testing.allocator, "---\nscalar1\n...\nkey: value\n");
defer stream.deinit(testing.allocator);
try testing.expectEqual(@as(usize, 2), stream.documents.len);
try testing.expect(stream.documents[0].explicit);
try testing.expectEqualSlices(u8, "scalar1", rootKind(stream.documents[0].document).string);
try testing.expect(!stream.documents[1].explicit);
try testing.expectEqualSlices(u8, "value", (try stream.documents[1].document.ast.getValByPath(&.{.{ .key = "key" }})).kind.string);
}
test "extractStream: single bare document" {
if (comptime !build_options.lang_yaml) return error.SkipZigTest;
const stream = try extractStream(testing.allocator, "key: value\n");
defer stream.deinit(testing.allocator);
try testing.expectEqual(@as(usize, 1), stream.documents.len);
try testing.expect(!stream.documents[0].explicit);
}
test "extractStream: empty stream is one null document" {
if (comptime !build_options.lang_yaml) return error.SkipZigTest;
const stream = try extractStream(testing.allocator, "");
defer stream.deinit(testing.allocator);
try testing.expectEqual(@as(usize, 1), stream.documents.len);
try testing.expect(rootKind(stream.documents[0].document) == .null_);
}
test "extractStream: outerSpan lifts node spans into outer coordinates" {
if (comptime !build_options.lang_yaml) return error.SkipZigTest;
const src = "---\nfoo\n---\nbar\n";
const stream = try extractStream(testing.allocator, src);
defer stream.deinit(testing.allocator);
try testing.expectEqual(@as(usize, 2), stream.documents.len);
const d1 = stream.documents[1];
const node = d1.document.ast.nodes[d1.document.ast.root];
const outer = d1.outerSpan(d1.document.span(node));
try testing.expectEqualSlices(u8, "bar", src[outer.start..outer.end]);
}
// --- Embed.detect / Embed.retype ------------------------------------------
test "detect: recognizes YAML frontmatter" {
const src = "---\ntitle: hi\n---\nbody\n";
try testing.expectEqual(@as(?Type, .{ .frontmatter = .yaml }), detect(src));
}
test "detect: recognizes JSON frontmatter" {
const src = ";;;\n{\"title\":\"hi\"}\n;;;\nbody\n";
try testing.expectEqual(@as(?Type, .semicolons_json), detect(src));
}
test "detect: recognizes fig frontmatter" {
if (comptime !build_options.lang_fig) return error.SkipZigTest;
const src = "```fig\ntitle = hi\n```\nbody\n";
try testing.expectEqual(@as(?Type, .{ .fenced = .fig }), detect(src));
}
test "detect: recognizes YAML endmatter" {
const src = "body prose\n```endmatter\ntitle: hi\n```\n";
try testing.expectEqual(@as(?Type, .endmatter_yaml), detect(src));
}
test "detect: an unterminated block is still recognized (caller sees Unterminated, not NotFound)" {
const src = "---\ntitle: hi\n";
try testing.expectEqual(@as(?Type, .{ .frontmatter = .yaml }), detect(src));
try testing.expectError(Error.Unterminated, locateRegion(src, detect(src).?));
}
test "detect: plain prose with no fences at all detects nothing" {
try testing.expectEqual(@as(?Type, null), detect("just some markdown\n\nno frontmatter here\n"));
}
test "retype: YAML frontmatter -> JSON frontmatter, body preserved byte-identical" {
const src = "---\ntitle: hi\n---\n# body\n";
const region = try locateRegion(src, .{ .frontmatter = .yaml });
const out = try retype(testing.allocator, src, region, .{ .frontmatter = .yaml }, .semicolons_json, "{\"title\":\"hi\"}\n");
defer testing.allocator.free(out);
try testing.expectEqualStrings(";;;\n{\"title\":\"hi\"}\n;;;\n# body\n", out);
}
test "retype: frontmatter -> endmatter moves the fences to the end, body first" {
const src = "---\ntitle: hi\n---\n# body\n";
const region = try locateRegion(src, .{ .frontmatter = .yaml });
const out = try retype(testing.allocator, src, region, .{ .frontmatter = .yaml }, .endmatter_yaml, "title: hi\n");
defer testing.allocator.free(out);
try testing.expectEqualStrings("# body\n```endmatter\ntitle: hi\n```\n", out);
}
/// Every archetype, on a source that has host text on BOTH sides of the block
/// plus a BOM — the shape that used to lose bytes. `locate` must account for
/// every byte exactly once (`Region`'s partition invariant), which is what
/// makes the `retype` rebuilds below lossless.
const tiling_cases = [_]struct { t: Type, src: []const u8 }{
.{ .t = .{ .frontmatter = .yaml }, .src = "\xEF\xBB\xBF---\nk: v\n---\nafter\n" },
.{ .t = .{ .fenced = .yaml }, .src = "```yaml\nk: v\n```\nafter\n" },
.{ .t = .semicolons_json, .src = ";;;\n{}\n;;;\nafter\n" },
.{ .t = .plus_toml, .src = "+++\nk = 1\n+++\nafter\n" },
.{ .t = .endmatter_yaml, .src = "before\n```endmatter\nk: v\n```\nafter\n" },
.{ .t = .{ .html_script = .yaml }, .src = "<head>\n<script type=\"application/yaml\">\nk: v\n</script>\n</head>\n" },
.{ .t = .{ .html_code = .yaml }, .src = "<p>x</p>\n<pre><code class=\"language-yaml\">\nk: v\n</code></pre>\n<p>y</p>\n" },
};
test "locate: the region and both body sides tile the source exactly" {
for (tiling_cases) |case| {
const region = try locateRegion(case.src, case.t);
try testing.expect(regionTilesSource(region, case.src));
// Spelled out once concretely: reassembling the five spans in order
// reproduces the file byte-for-byte, BOM included.
var buf: std.ArrayList(u8) = .empty;
defer buf.deinit(testing.allocator);
for ([_]Span{ region.body_before, region.open_fence, region.content, region.close_fence, region.body_after }) |s|
try buf.appendSlice(testing.allocator, Span.of(u8, s, case.src));
try testing.expectEqualStrings(case.src, buf.items);
}
}
test "retype: a same-archetype rebuild is byte-identical for every archetype" {
// The strongest statement of losslessness available without a parser: retype
// an archetype to ITSELF, re-housing its own content unchanged. Any host
// byte the rebuild drops — a BOM, a prefix, a trailing line — shows up here
// as a diff. Each of these used to fail on one case or another.
for (tiling_cases) |case| {
const region = try locateRegion(case.src, case.t);
const inner = Span.of(u8, region.content, case.src);
const out = try retype(testing.allocator, case.src, region, case.t, case.t, inner);
defer testing.allocator.free(out);
try testing.expectEqualStrings(case.src, out);
}
}
test "retype: a BOM stays at offset 0 when the block moves" {
const src = "\xEF\xBB\xBF---\ntitle: hi\n---\n# body\n";
const region = try locateRegion(src, .{ .frontmatter = .yaml });
// Frontmatter -> endmatter sends the block to the bottom; the BOM must not
// travel with the prose it happens to precede, or it stops being a BOM.
const out = try retype(testing.allocator, src, region, .{ .frontmatter = .yaml }, .endmatter_yaml, "title: hi\n");
defer testing.allocator.free(out);
try testing.expectEqualStrings("\xEF\xBB\xBF# body\n```endmatter\ntitle: hi\n```\n", out);
}
test "retype: endmatter keeps text that trailed the close fence" {
const src = "prose\n```endmatter\ntitle: hi\n```\ntrailing\n";
const region = try locateRegion(src, .endmatter_yaml);
const out = try retype(testing.allocator, src, region, .endmatter_yaml, .{ .fenced = .yaml }, "title: hi\n");
defer testing.allocator.free(out);
// Both sides of the old block survive, in order, below the new fence.
try testing.expectEqualStrings("```yaml\ntitle: hi\n```\nprose\ntrailing\n", out);
}
test "retype: a mid-document block re-houses in place, both sides intact" {
const src = "<head>\n<script type=\"application/yaml\">\nk: v\n</script>\n</head>\n";
const region = try locateRegion(src, .{ .html_script = .yaml });
const out = try retype(testing.allocator, src, region, .{ .html_script = .yaml }, .{ .html_code = .yaml }, "k: v\n");
defer testing.allocator.free(out);
try testing.expectEqualStrings(
"<head>\n<pre><code class=\"language-yaml\">\nk: v\n</code></pre>\n</head>\n",
out,
);
}
test "retype: refuses to move a mid-document block to an edge archetype" {
// Hoisting a `---` fence above `<head>` is neither valid markdown nor valid
// HTML, and leaving it put is not frontmatter. Refuse rather than pick one.
const src = "<head>\n<script type=\"application/yaml\">\nk: v\n</script>\n</head>\n";
const region = try locateRegion(src, .{ .html_script = .yaml });
for ([_]Type{ .{ .frontmatter = .yaml }, .{ .fenced = .yaml }, .semicolons_json, .plus_toml, .endmatter_yaml }) |to| {
try testing.expectError(
error.MidDocumentRegionCannotMove,
retype(testing.allocator, src, region, .{ .html_script = .yaml }, to, "k: v\n"),
);
}
}
test "initRegion: a BOM'd source keeps its BOM at offset 0" {
const init = try initRegion(testing.allocator, "\xEF\xBB\xBFbody text\n", .plus_toml);
defer testing.allocator.free(init.host);
// The block goes after the BOM, not before it.
try testing.expectEqualStrings("\xEF\xBB\xBF+++\n+++\nbody text\n", init.host);
try testing.expectEqualStrings("body text\n", Span.of(u8, init.region.body, init.host));
}