//! A stable, inspectable JSON encoding of the shared `AST`, for debugging
//! parsers and diffing tree shapes across runs. This is a LIBRARY module
//! (exported from `root.zig` as `twig.ast_json`) so every surface can reach
//! it: `twig convert -o ast` (`cli/actions.zig`) and the C ABI's
//! `twig_document_ast_json` (`c_abi.zig`) both call `encode`/`encodeAlloc`
//! here rather than each carrying their own encoder.
//!
//! Every node becomes an object with a `"kind"` tag (the `Node.Kind` union's
//! tag name, e.g. `"heading"`, `"str"`), a `"span"` byte range, that kind's
//! own payload fields inlined (switching exhaustively over `AST.Node.Kind` —
//! see `writeKindPayload`), and `"attrs"`/`"children"` when non-empty.
//!
//! Escaping correctness comes for free from `std.json.Stringify.write`,
//! which is used for every leaf value (strings, ints, bools, `?T`, and even
//! the payload enums like `ListNumbering`/`Alignment`, which `write`
//! renders as their tag name — no manual `@tagName` calls needed for those).
//! Only the *shape* (which fields a kind gets, in what order) is decided by
//! hand below, via `writeNode`'s explicit `beginObject`/`objectField`/
//! `endObject` calls walking the `first_child`/`next_sibling` tree — `write`
//! alone can't do that part since `AST` is a linked structure, not a
//! `[]node`-shaped value `write`'s reflection could walk on its own.
const std = @import("std");
const Allocator = std.mem.Allocator;
const Writer = std.Io.Writer;
const Stringify = std.json.Stringify;
const AST = @import("ast.zig");
const Document = @import("../document.zig");
const Node = AST.Node;
/// Encode `ast` (rooted at `ast.root`) as pretty-printed (2-space indent)
/// JSON, writing to `writer`. Emits a trailing newline so piping straight to
/// a terminal or a file looks like any other well-behaved text tool's
/// output.
pub fn encode(doc: *const Document, writer: *Writer) Writer.Error!void {
var w: Stringify = .{ .writer = writer, .options = .{ .whitespace = .indent_2 } };
try writeNode(&w, doc, doc.ast.root);
try writer.writeByte('\n');
}
/// `encode` into a freshly allocated, caller-owned buffer. The underlying
/// `Writer.Allocating` only ever fails on allocation, so the encoder's
/// `Writer.Error` collapses to `Allocator.Error` here.
pub fn encodeAlloc(allocator: Allocator, doc: *const Document) Allocator.Error![]u8 {
var out: Writer.Allocating = .init(allocator);
defer out.deinit();
encode(doc, &out.writer) catch |err| switch (err) {
error.WriteFailed => return error.OutOfMemory,
};
return out.toOwnedSlice();
}
fn writeNode(w: *Stringify, doc: *const Document, id: Node.Id) Writer.Error!void {
const node = doc.ast.nodes[id];
try w.beginObject();
try w.objectField("kind");
try w.write(node.kind.kindName());
try w.objectField("span");
try w.beginArray();
try w.write(doc.span(id).start);
try w.write(doc.span(id).end);
try w.endArray();
if (doc.contentSpan(id)) |cs| {
try w.objectField("content_span");
try w.beginArray();
try w.write(cs.start);
try w.write(cs.end);
try w.endArray();
}
if (doc.markerSpan(id)) |ms| {
try w.objectField("marker_span");
try w.beginArray();
try w.write(ms.start);
try w.write(ms.end);
try w.endArray();
}
try writeKindPayload(w, node.kind);
const attrs = doc.ast.attrsOf(id);
if (!attrs.isEmpty()) {
try w.objectField("attrs");
try w.beginArray();
for (attrs.entries) |kv| {
try w.beginObject();
try w.objectField("key");
try w.write(kv.key);
try w.objectField("value");
try w.write(kv.value);
try w.endObject();
}
try w.endArray();
}
if (node.first_child != null) {
try w.objectField("children");
try w.beginArray();
var it = doc.children(id);
while (it.next()) |child| try writeNode(w, doc, child.id);
try w.endArray();
}
try w.endObject();
}
/// Write the fields specific to `kind`'s payload — the part of each node
/// that isn't `kind`/`span`/`content_span`/`attrs`/`children`. Switches
/// exhaustively over `AST.Node.Kind` (see `ast.zig`'s doc comment for the
/// full vocabulary) so adding a new `Kind` variant fails this file's build
/// until it's given a field mapping here.
///
/// Public because it is also the payload half of the node table
/// (`ast/table.zig`), whose rows carry exactly these fields; `readKind` below
/// is the same switch read the other way, so a new kind fails both.
pub fn writeKindPayload(w: *Stringify, kind: Node.Kind) Writer.Error!void {
switch (kind) {
// Payload-free kinds: nothing beyond kind/span/attrs/children.
.doc,
.para,
.thematic_break,
.section,
.block_quote,
.definition_list,
.line_block,
.table,
.list_item,
.definition_list_item,
.term,
.definition,
// Its width/stub data lives in `attrs`, which is written for every
// node — see `Kind.column`.
.column,
.caption,
.soft_break,
.hard_break,
.non_breaking_space,
// `inline_mark`'s family member is already reported as the node's
// `kind` name (see `Kind.kindName`), so it needs no payload field.
.inline_mark,
=> {},
.heading => |h| {
try w.objectField("level");
try w.write(h.level);
},
.code_block => |c| {
try w.objectField("lang");
try w.write(c.lang);
try w.objectField("text");
try w.write(c.text);
},
.raw_block => |r| {
try w.objectField("format");
try w.write(r.format);
try w.objectField("text");
try w.write(r.text);
},
.metadata => |m| {
try w.objectField("lang");
try w.write(m.lang);
try w.objectField("text");
try w.write(m.text);
},
.bullet_list => |b| {
try w.objectField("tight");
try w.write(b.tight);
},
.ordered_list => |o| {
try w.objectField("numbering");
try w.write(o.numbering);
try w.objectField("tight");
try w.write(o.tight);
try w.objectField("start");
try w.write(o.start);
},
.task_list => |t| {
try w.objectField("tight");
try w.write(t.tight);
},
.task_list_item => |t| {
try w.objectField("checked");
try w.write(t.checked);
},
.line => |l| {
try w.objectField("indent");
try w.write(l.indent);
},
.row => |r| {
try w.objectField("head");
try w.write(r.head);
},
.cell => |c| {
try w.objectField("head");
try w.write(c.head);
try w.objectField("alignment");
try w.write(c.alignment);
try w.objectField("colspan");
try w.write(c.colspan);
try w.objectField("rowspan");
try w.write(c.rowspan);
},
.footnote => |f| {
try w.objectField("label");
try w.write(f.label);
},
// The other two named definitions carry the same one field, under the
// same name — a consumer that reads a footnote's label reads these
// without a second code path, and the registry it resolves in is the
// node's `kind`.
.citation => |c| {
try w.objectField("label");
try w.write(c.label);
},
.substitution => |s| {
try w.objectField("label");
try w.write(s.label);
},
.reference => |r| {
try w.objectField("label");
try w.write(r.label);
try w.objectField("destination");
try w.write(r.destination);
},
.str => |s| {
try w.objectField("text");
try w.write(s);
},
// The seven text leaves report their family member as the node's
// `kind` name (see `Kind.kindName`), so one arm serves all of them.
.text_leaf => |l| {
try w.objectField("text");
try w.write(l.text);
},
.raw_inline => |r| {
try w.objectField("format");
try w.write(r.format);
try w.objectField("text");
try w.write(r.text);
},
.smart_punctuation => |sp| {
try w.objectField("punctuation_kind");
try w.write(sp);
// No stored spelling to report anymore (see `Kind.smart_punctuation`'s
// doc) — `text` is derived so the published JSON vocabulary is
// unchanged for existing consumers.
try w.objectField("text");
try w.write(sp.ascii());
},
.link => |l| {
try w.objectField("destination");
try w.write(l.destination);
try w.objectField("reference");
try w.write(l.reference);
},
.image => |l| {
try w.objectField("destination");
try w.write(l.destination);
try w.objectField("reference");
try w.write(l.reference);
},
.container => |c| {
try w.objectField("name");
try w.write(c.name);
try w.objectField("form");
try w.write(c.form);
try w.objectField("argument");
try w.write(c.argument);
// Present only where the body was read as text, so a consumer
// walking `children` on every other container sees no new key.
if (c.text) |t| {
try w.objectField("text");
try w.write(t);
}
},
// The three markup leaves report their family member as the node's
// `kind` name (see `Kind.kindName`), so one arm serves all of them.
.markup_leaf => |l| {
try w.objectField("text");
try w.write(l.text);
},
.processing_instruction => |p| {
try w.objectField("target");
try w.write(p.target);
try w.objectField("data");
try w.write(p.data);
},
}
}
/// Why `readKind` refused an object: which field, and what was wrong with it.
/// A decoder over input it did not write — a runtime language's table — has
/// to name the field, because the author has nothing else to go on.
pub const ReadError = error{InvalidPayload};
pub const FieldProblem = struct {
field: []const u8 = "",
what: []const u8 = "",
};
/// The kind `ref` names, with its payload read from `obj` — the inverse of
/// `writeKindPayload`, switching over the same arms. Strings in the result
/// BORROW from `obj`; `AST.Builder.addNode` copies them.
///
/// A field whose Zig type has a default (`Cell.colspan`, `Line.indent`) or is
/// optional may be absent; every other field is required. An enum payload is
/// read by its tag name, which is what `writeKindPayload` writes.
pub fn readKind(ref: AST.KindRef, obj: std.json.ObjectMap, problem: *FieldProblem) ReadError!Node.Kind {
const r: Reader = .{ .obj = obj, .problem = problem };
return switch (ref) {
.mark => |m| .{ .inline_mark = m },
.text_leaf => |k| .{ .text_leaf = .{ .kind = k, .text = try r.str("text") } },
.markup_leaf => |k| .{ .markup_leaf = .{ .kind = k, .text = try r.str("text") } },
.container_named => r.fail("kind", "not a published kind name"),
.tag => |tag| switch (tag) {
.inline_mark, .text_leaf, .markup_leaf => r.fail("kind", "a family is not a published kind name"),
.doc => .doc,
.para => .para,
.thematic_break => .thematic_break,
.section => .section,
.block_quote => .block_quote,
.definition_list => .definition_list,
.line_block => .line_block,
.table => .table,
.list_item => .list_item,
.definition_list_item => .definition_list_item,
.term => .term,
.definition => .definition,
.column => .column,
.caption => .caption,
.soft_break => .soft_break,
.hard_break => .hard_break,
.non_breaking_space => .non_breaking_space,
.heading => .{ .heading = .{ .level = try r.int("level") } },
.code_block => .{ .code_block = .{ .lang = try r.optStr("lang"), .text = try r.str("text") } },
.raw_block => .{ .raw_block = .{ .format = try r.str("format"), .text = try r.str("text") } },
.metadata => .{ .metadata = .{ .lang = try r.str("lang"), .text = try r.str("text") } },
.bullet_list => .{ .bullet_list = .{ .tight = try r.boolean("tight") } },
.ordered_list => .{ .ordered_list = .{
.numbering = try r.enumeration(AST.ListNumbering, "numbering"),
.tight = try r.boolean("tight"),
.start = try r.optInt("start"),
} },
.task_list => .{ .task_list = .{ .tight = try r.boolean("tight") } },
.task_list_item => .{ .task_list_item = .{ .checked = try r.boolean("checked") } },
.line => .{ .line = .{ .indent = try r.optInt("indent") orelse 0 } },
.row => .{ .row = .{ .head = try r.boolean("head") } },
.cell => .{ .cell = .{
.head = try r.boolean("head"),
.alignment = try r.enumeration(AST.Alignment, "alignment"),
.colspan = try r.optInt("colspan") orelse 1,
.rowspan = try r.optInt("rowspan") orelse 1,
} },
.footnote => .{ .footnote = .{ .label = try r.str("label") } },
.citation => .{ .citation = .{ .label = try r.str("label") } },
.substitution => .{ .substitution = .{ .label = try r.str("label") } },
.reference => .{ .reference = .{ .label = try r.str("label"), .destination = try r.str("destination") } },
.str => .{ .str = try r.str("text") },
.raw_inline => .{ .raw_inline = .{ .format = try r.str("format"), .text = try r.str("text") } },
// `text` is written beside it for readers, and derived; the kind is
// the fact.
.smart_punctuation => .{ .smart_punctuation = try r.enumeration(AST.SmartPunctuationKind, "punctuation_kind") },
.link => .{ .link = .{ .destination = try r.optStr("destination"), .reference = try r.optStr("reference") } },
.image => .{ .image = .{ .destination = try r.optStr("destination"), .reference = try r.optStr("reference") } },
.container => .{ .container = .{
.name = try r.str("name"),
.form = try r.optEnumeration(AST.Form, "form"),
.argument = try r.optStr("argument"),
.text = try r.optStr("text"),
} },
.processing_instruction => .{ .processing_instruction = .{ .target = try r.str("target"), .data = try r.str("data") } },
},
};
}
/// Typed field reads over one JSON object, each naming its field on failure.
const Reader = struct {
obj: std.json.ObjectMap,
problem: *FieldProblem,
fn fail(self: Reader, field: []const u8, what: []const u8) ReadError {
self.problem.* = .{ .field = field, .what = what };
return error.InvalidPayload;
}
/// The value at `field`, with an explicit JSON `null` read as absent.
fn get(self: Reader, field: []const u8) ?std.json.Value {
const v = self.obj.get(field) orelse return null;
return if (v == .null) null else v;
}
fn str(self: Reader, field: []const u8) ReadError![]const u8 {
return try self.optStr(field) orelse self.fail(field, "required string is missing");
}
fn optStr(self: Reader, field: []const u8) ReadError!?[]const u8 {
const v = self.get(field) orelse return null;
return switch (v) {
.string => |s| s,
else => self.fail(field, "expected a string"),
};
}
fn int(self: Reader, field: []const u8) ReadError!u32 {
return try self.optInt(field) orelse self.fail(field, "required integer is missing");
}
fn optInt(self: Reader, field: []const u8) ReadError!?u32 {
const v = self.get(field) orelse return null;
return switch (v) {
.integer => |i| std.math.cast(u32, i) orelse self.fail(field, "integer out of range"),
else => self.fail(field, "expected an integer"),
};
}
fn boolean(self: Reader, field: []const u8) ReadError!bool {
const v = self.get(field) orelse return self.fail(field, "required boolean is missing");
return switch (v) {
.bool => |b| b,
else => self.fail(field, "expected a boolean"),
};
}
fn enumeration(self: Reader, comptime E: type, field: []const u8) ReadError!E {
return try self.optEnumeration(E, field) orelse self.fail(field, "required name is missing");
}
fn optEnumeration(self: Reader, comptime E: type, field: []const u8) ReadError!?E {
const name = try self.optStr(field) orelse return null;
return std.meta.stringToEnum(E, name) orelse self.fail(field, "not one of the names this field takes");
}
};
const testing = std.testing;
test "encode: leaf node gets kind/span, omits content_span/attrs/children when absent" {
var b = AST.Builder.init(testing.allocator);
defer b.deinit();
const leaf = try b.addLeaf(.{ .str = "hi" });
b.setSpan(leaf, .init(0, 2));
var ast = try b.finishDocument("", leaf);
defer ast.deinit();
const out = try encodeAlloc(testing.allocator, &ast);
defer testing.allocator.free(out);
var parsed = try std.json.parseFromSlice(std.json.Value, testing.allocator, out, .{});
defer parsed.deinit();
const obj = parsed.value.object;
try testing.expectEqualStrings("str", obj.get("kind").?.string);
try testing.expectEqualStrings("hi", obj.get("text").?.string);
const span = obj.get("span").?.array;
try testing.expectEqual(@as(usize, 2), span.items.len);
try testing.expectEqual(@as(i64, 0), span.items[0].integer);
try testing.expectEqual(@as(i64, 2), span.items[1].integer);
try testing.expectEqual(@as(?std.json.Value, null), obj.get("content_span"));
try testing.expectEqual(@as(?std.json.Value, null), obj.get("attrs"));
try testing.expectEqual(@as(?std.json.Value, null), obj.get("children"));
}
test "encode: container node nests children in source order" {
var b = AST.Builder.init(testing.allocator);
defer b.deinit();
const a = try b.addLeaf(.{ .str = "a" });
const em_text = try b.addLeaf(.{ .str = "b" });
const em = try b.addContainer(.{ .inline_mark = .emph }, &.{em_text});
const para = try b.addContainer(.para, &.{ a, em });
var ast = try b.finishDocument("", para);
defer ast.deinit();
const out = try encodeAlloc(testing.allocator, &ast);
defer testing.allocator.free(out);
var parsed = try std.json.parseFromSlice(std.json.Value, testing.allocator, out, .{});
defer parsed.deinit();
const obj = parsed.value.object;
try testing.expectEqualStrings("para", obj.get("kind").?.string);
const children = obj.get("children").?.array;
try testing.expectEqual(@as(usize, 2), children.items.len);
try testing.expectEqualStrings("str", children.items[0].object.get("kind").?.string);
try testing.expectEqualStrings("a", children.items[0].object.get("text").?.string);
try testing.expectEqualStrings("emph", children.items[1].object.get("kind").?.string);
const em_children = children.items[1].object.get("children").?.array;
try testing.expectEqualStrings("b", em_children.items[0].object.get("text").?.string);
}
test "encode: attrs render as ordered key/value pairs, bare attrs get a null value" {
var b = AST.Builder.init(testing.allocator);
defer b.deinit();
const el = try b.addLeaf(.{ .container = .{ .name = "input" } });
try b.setAttrs(el, .{ .entries = &.{
.{ .key = "disabled", .value = null },
.{ .key = "type", .value = "checkbox" },
} });
var ast = try b.finishDocument("", el);
defer ast.deinit();
const out = try encodeAlloc(testing.allocator, &ast);
defer testing.allocator.free(out);
var parsed = try std.json.parseFromSlice(std.json.Value, testing.allocator, out, .{});
defer parsed.deinit();
const obj = parsed.value.object;
try testing.expectEqualStrings("container", obj.get("kind").?.string);
try testing.expectEqualStrings("input", obj.get("name").?.string);
const attrs = obj.get("attrs").?.array;
try testing.expectEqual(@as(usize, 2), attrs.items.len);
try testing.expectEqualStrings("disabled", attrs.items[0].object.get("key").?.string);
try testing.expectEqual(std.json.Value.null, attrs.items[0].object.get("value").?);
try testing.expectEqualStrings("type", attrs.items[1].object.get("key").?.string);
try testing.expectEqualStrings("checkbox", attrs.items[1].object.get("value").?.string);
}
test "encode: content_span is emitted only when set, and enum payloads render as tag-name strings" {
var b = AST.Builder.init(testing.allocator);
defer b.deinit();
const list = try b.addContainer(.{ .ordered_list = .{ .numbering = .lower_alpha, .tight = true, .start = null } }, &.{});
b.setContentSpan(list, .init(1, 5));
var ast = try b.finishDocument("", list);
defer ast.deinit();
const out = try encodeAlloc(testing.allocator, &ast);
defer testing.allocator.free(out);
var parsed = try std.json.parseFromSlice(std.json.Value, testing.allocator, out, .{});
defer parsed.deinit();
const obj = parsed.value.object;
try testing.expectEqualStrings("ordered_list", obj.get("kind").?.string);
try testing.expectEqualStrings("lower_alpha", obj.get("numbering").?.string);
try testing.expectEqual(true, obj.get("tight").?.bool);
// The marker spelling (`- ` vs `* `, `1.` vs `1)`) left the AST for the
// Document's spelling table, so the encoding no longer carries it.
try testing.expectEqual(@as(?std.json.Value, null), obj.get("style"));
try testing.expectEqual(@as(?std.json.Value, null), obj.get("delim"));
const cs = obj.get("content_span").?.array;
try testing.expectEqual(@as(i64, 1), cs.items[0].integer);
try testing.expectEqual(@as(i64, 5), cs.items[1].integer);
}
test "encode is pretty-printed with 2-space indentation" {
var b = AST.Builder.init(testing.allocator);
defer b.deinit();
const leaf = try b.addLeaf(.{ .str = "x" });
const root = try b.addContainer(.para, &.{leaf});
var ast = try b.finishDocument("", root);
defer ast.deinit();
const out = try encodeAlloc(testing.allocator, &ast);
defer testing.allocator.free(out);
try testing.expect(std.mem.indexOf(u8, out, "\n \"kind\"") != null);
}