//! Djot: the entry point for this language module. Wires the block scanner,
//! inline scanner, and event-stream-to-AST builder together into one
//! `parse` call, and aggregates every sibling file's `test {}` blocks (the
//! per-language-module convention shared with sister project `fig`; see
//! DESIGN.md's "Relationship to fig").
//!
//! Block/inline scanning is naturally two cooperating files rather than
//! fig's usual single `tokenizer.zig`, since Djot's block and inline levels
//! are mutually interleaved (each paragraph/heading gets its own inline scan
//! as the block scanner reaches its content) — see `event.zig`'s doc
//! comment for the full picture of how the pieces fit together.
//!
//! This is also where everything djot-specific-but-not-parse-time lives:
//! `Document` (the shared `AST` plus djot's reference-resolution side
//! tables) and the `isBlock`/`isInline` classification of the shared kind
//! vocabulary (djot's block/inline dichotomy is meaningless for, say, a
//! generic XML `element`, so it has no business in `ast/`).
const std = @import("std");
const Allocator = std.mem.Allocator;
const block = @import("block.zig");
const inline_mod = @import("inline.zig");
const parser = @import("parser.zig");
pub const AST = @import("../../ast/ast.zig");
pub const html = @import("html.zig");
pub const serializer = @import("serializer.zig");
/// A parsed djot document: the language-neutral `AST` plus the label ->
/// definition-node maps djot's render-time reference resolution needs.
/// These are side tables of the AST proper (their keys live in the AST's
/// `owned_strings`, and their values are plain node ids), split out so the
/// shared `AST` stays free of djot-only baggage — XML/HTML have nothing
/// like them.
pub const Document = struct {
ast: AST,
/// Label (normalized) -> the `reference` definition node with that label.
/// Populated during parsing; never consulted during parsing itself —
/// label resolution (matching a `link`/`image`'s `reference` string
/// against this table) is deferred entirely to render time, so forward
/// references need no special handling. See djot.js's
/// `parse.ts`/`html.ts` split.
references: std.StringHashMapUnmanaged(AST.Node.Id) = .empty,
/// Same shape as `references`, for reference definitions synthesized by
/// the parser itself (e.g. implicit heading-derived link targets) rather
/// than written explicitly by the author.
auto_references: std.StringHashMapUnmanaged(AST.Node.Id) = .empty,
/// Label -> the `footnote` definition node with that label.
footnotes: std.StringHashMapUnmanaged(AST.Node.Id) = .empty,
pub fn deinit(self: *Document) void {
// The maps' keys live in `ast.owned_strings`; only the map
// structures themselves are freed here (before the strings go away,
// though the order is immaterial — map deinit never reads keys).
self.references.deinit(self.ast.allocator);
self.auto_references.deinit(self.ast.allocator);
self.footnotes.deinit(self.ast.allocator);
self.ast.deinit();
}
};
/// Parse `source` (Djot markup) into a `Document`. The returned document is
/// fully self-contained (its AST owns copies of every string it needs) and
/// must be freed with `doc.deinit()`.
pub fn parse(allocator: Allocator, source: []const u8) Allocator.Error!Document {
var block_parser = try block.Parser.init(allocator, source);
defer block_parser.deinit();
const events = try block_parser.scan();
defer allocator.free(events);
var tree_builder = parser.TreeBuilder.init(allocator, block_parser.subject);
return tree_builder.build(events);
}
// ── block/inline classification ─────────────────────────────────────────
// Djot's view of the shared kind vocabulary; the generic-markup kinds
// (`element`, `comment`, ...) never appear in a djot parse and are in
// neither set.
const block_tags = std.EnumSet(std.meta.Tag(AST.Node.Kind)).initMany(&.{
.para, .heading, .thematic_break, .section, .div,
.code_block, .raw_block, .block_quote, .bullet_list, .ordered_list,
.task_list, .definition_list, .table, .reference, .footnote,
});
const inline_tags = std.EnumSet(std.meta.Tag(AST.Node.Kind)).initMany(&.{
.str, .soft_break, .hard_break, .non_breaking_space, .symb,
.verbatim, .raw_inline, .inline_math, .display_math, .url,
.email, .footnote_reference, .smart_punctuation, .emph, .strong,
.link, .image, .span, .mark, .superscript,
.subscript, .insert, .delete, .double_quoted, .single_quoted,
});
/// Mirrors djot.js `ast.ts`'s `isBlock`.
pub fn isBlock(kind: AST.Node.Kind) bool {
return block_tags.contains(std.meta.activeTag(kind));
}
/// Mirrors djot.js `ast.ts`'s `isInline`.
pub fn isInline(kind: AST.Node.Kind) bool {
return inline_tags.contains(std.meta.activeTag(kind));
}
pub const AutolinkKind = inline_mod.InlineParser.AutolinkKind;
/// What `<dest>` (given here WITHOUT its angle brackets) would spell if it
/// appeared in a djot document — `null` if djot would leave it literal text.
/// The inline scanner's own classifier, so a caller asking "may I spell this
/// destination as an autolink?" gets the answer the reparse will give.
pub const autolinkKindOf = inline_mod.InlineParser.autolinkKindOf;
test {
_ = @import("event.zig");
_ = @import("attributes.zig");
_ = @import("block.zig");
_ = @import("inline.zig");
_ = @import("parser.zig");
_ = @import("html.zig");
_ = @import("conformance.zig");
}
const testing = std.testing;
test "parse produces a doc with a paragraph" {
var doc = try parse(testing.allocator, "hello *world*\n");
defer doc.deinit();
const ast = doc.ast;
try testing.expect(ast.nodes[ast.root].kind == .doc);
const para_id = ast.nodes[ast.root].first_child orelse return error.TestExpectedNonNull;
try testing.expect(ast.nodes[para_id].kind == .para);
const str_id = ast.nodes[para_id].first_child orelse return error.TestExpectedNonNull;
try testing.expectEqualStrings("hello ", ast.nodes[str_id].kind.str);
const strong_id = ast.nodes[str_id].next_sibling orelse return error.TestExpectedNonNull;
try testing.expect(ast.nodes[strong_id].kind == .strong);
}
test "heading gets an auto id and wraps a section" {
var doc = try parse(testing.allocator, "# Hello World\n\npara\n");
defer doc.deinit();
const ast = doc.ast;
const section_id = ast.nodes[ast.root].first_child orelse return error.TestExpectedNonNull;
try testing.expect(ast.nodes[section_id].kind == .section);
const attrs = ast.attrsOf(section_id);
try testing.expectEqualStrings("Hello-World", attrs.get("id").?);
const heading_id = ast.nodes[section_id].first_child orelse return error.TestExpectedNonNull;
try testing.expect(ast.nodes[heading_id].kind.heading.level == 1);
}
test "reference link resolves via the references map" {
var doc = try parse(testing.allocator,
\\[foo][bar]
\\
\\[bar]: http://example.com
\\
);
defer doc.deinit();
try testing.expect(doc.references.contains("bar"));
}
test "bullet list is tight, definition list restructures term/definition" {
var doc = try parse(testing.allocator, "- a\n- b\n");
defer doc.deinit();
const list_id = doc.ast.nodes[doc.ast.root].first_child orelse return error.TestExpectedNonNull;
try testing.expect(doc.ast.nodes[list_id].kind.bullet_list.tight);
var doc2 = try parse(testing.allocator, "orange\n\n: a citrus fruit\n");
defer doc2.deinit();
}
test "isBlock/isInline classify kinds" {
try testing.expect(isBlock(.{ .heading = .{ .level = 1 } }));
try testing.expect(!isInline(.{ .heading = .{ .level = 1 } }));
try testing.expect(isInline(.{ .str = "x" }));
try testing.expect(!isBlock(.{ .str = "x" }));
// Generic-markup kinds are neither: djot never produces them.
try testing.expect(!isBlock(.{ .element = .{ .name = "video" } }));
try testing.expect(!isInline(.{ .element = .{ .name = "video" } }));
}
// ── djot.js AST-dump-only cases, asserted natively ──────────────────────────
// The djot.js corpus has 6 cases whose expected output is djot.js's internal
// AST-dump debug format rather than HTML, so the HTML conformance run skips
// them (see conformance.zig). These tests assert the same parser behaviours
// directly against Twig's own AST, so those behaviours are covered and the
// "100% djot conformant" claim is honest. Verified against `renderAST` output
// in the corpus: symb.test, attributes.test, regression.test, sourcepos.test.
test "symb: :name: shortcodes parse to symb nodes carrying the bare alias" {
var doc = try parse(testing.allocator, ":+1: :scream:\n");
defer doc.deinit();
const ast = doc.ast;
const para = ast.nodes[ast.root].first_child orelse return error.TestExpectedNonNull;
const first = ast.nodes[para].first_child orelse return error.TestExpectedNonNull;
try testing.expect(ast.nodes[first].kind == .symb);
try testing.expectEqualStrings("+1", ast.nodes[first].kind.symb);
const space = ast.nodes[first].next_sibling orelse return error.TestExpectedNonNull;
try testing.expect(ast.nodes[space].kind == .str);
const second = ast.nodes[space].next_sibling orelse return error.TestExpectedNonNull;
try testing.expect(ast.nodes[second].kind == .symb);
try testing.expectEqualStrings("scream", ast.nodes[second].kind.symb);
}
test "symb: a shortcode consumes only through its closing colon, leaving the rest literal" {
var doc = try parse(testing.allocator, ":ice:scream:\n");
defer doc.deinit();
const ast = doc.ast;
const para = ast.nodes[ast.root].first_child orelse return error.TestExpectedNonNull;
const first = ast.nodes[para].first_child orelse return error.TestExpectedNonNull;
try testing.expect(ast.nodes[first].kind == .symb);
try testing.expectEqualStrings("ice", ast.nodes[first].kind.symb);
// ":ice:" is consumed; the trailing "scream:" stays literal text.
const rest = ast.nodes[first].next_sibling orelse return error.TestExpectedNonNull;
try testing.expect(ast.nodes[rest].kind == .str);
try testing.expectEqualStrings("scream:", ast.nodes[rest].kind.str);
}
test "attributes: a quoted value spanning multiple lines collapses to single spaces" {
var doc = try parse(testing.allocator,
\\{
\\ attr="long
\\ value
\\ spanning
\\ multiple
\\ lines"
\\ }
\\> a
\\
);
defer doc.deinit();
const ast = doc.ast;
const bq = ast.nodes[ast.root].first_child orelse return error.TestExpectedNonNull;
try testing.expect(ast.nodes[bq].kind == .block_quote);
try testing.expectEqualStrings(
"long value spanning multiple lines",
ast.attrsOf(bq).get("attr").?,
);
}
test "attributes: backslash escapes resolve inside a quoted value" {
var doc = try parse(testing.allocator,
\\> {key="bar
\\> a\$bim"}
\\> ou
\\
);
defer doc.deinit();
const ast = doc.ast;
const bq = ast.nodes[ast.root].first_child orelse return error.TestExpectedNonNull;
try testing.expect(ast.nodes[bq].kind == .block_quote);
const para = ast.nodes[bq].first_child orelse return error.TestExpectedNonNull;
try testing.expect(ast.nodes[para].kind == .para);
// The continuation collapses to one space and `\$` resolves to a literal $.
try testing.expectEqualStrings("bar a$bim", ast.attrsOf(para).get("key").?);
}
test "table: a later caption replaces an earlier one (djot.js issue #57)" {
var doc = try parse(testing.allocator,
\\| 1 | 2 |
\\
\\ ^ cap1
\\
\\ ^ cap2
\\
);
defer doc.deinit();
const ast = doc.ast;
const table = ast.nodes[ast.root].first_child orelse return error.TestExpectedNonNull;
try testing.expect(ast.nodes[table].kind == .table);
// A table's first child is always its caption; the later `^ cap2` wins.
const caption = ast.nodes[table].first_child orelse return error.TestExpectedNonNull;
try testing.expect(ast.nodes[caption].kind == .caption);
const str = ast.nodes[caption].first_child orelse return error.TestExpectedNonNull;
try testing.expect(ast.nodes[str].kind == .str);
try testing.expectEqualStrings("cap2", ast.nodes[str].kind.str);
}