//! The parser turns a TOML-formatted []const u8 into an AST.
//!
//! Phase 1: root-level `key = value` statements with scalar values (strings,
//! integers, floats, booleans, datetimes). Dotted keys and `[table]` headers
//! (Phase 2) and arrays / inline tables (Phase 3) are not assembled yet — a
//! statement that needs them returns error.NotImplemented.
//!
//! Scalars keep their source text verbatim (numbers/datetimes store `raw`;
//! decoded strings live in `owned_strings`). Number/datetime *normalization*
//! (e.g. 0xFF → 255 for JSON) is a print/convert concern, not done here.
const Parser = @This();
const std = @import("std");
const testing = std.testing;
const AST = @import("../../ast/ast.zig");
const Document = @import("../../document.zig");
const Type = @import("toml.zig").Type;
const util = @import("../../util/util.zig");
const Span = @import("../../util/span.zig");
const ascii = util.ascii;
const datetime = util.datetime;
const Unicode = util.Unicode;
const eqAny = util.eqlAny;
const Tokenizer = @import("tokenizer.zig");
const Token = Tokenizer.Token;
allocator: std.mem.Allocator,
version: Type = .TOML_1_0,
source: []const u8 = "",
tokens: []const Token = &.{},
pos: usize = 0,
nodes: std.ArrayList(AST.Node) = .empty,
spans: std.ArrayList(Span) = .empty,
owned_strings: std.ArrayList([]const u8) = .empty,
// Comment layer. Comments are captured in `skipInline`/`skipBlank` (the trivia
// skippers): a comment on the same line as a just-parsed value (`last_value_id`
// set) trails it; one on its own line buffers as `pending_leading`, claimed onto
// the next entry's key in `appendKeyValue`. A newline resets the trailing
// window. Text borrows `source`. `pending_leading` is reserved to `tokens.len`
// once so capture cannot fail. Materialized only when `comments_seen`.
node_comments: std.ArrayList(AST.NodeComments) = .empty,
pending_leading: std.ArrayList(AST.Comment) = .empty,
last_value_id: ?AST.Node.Id = null,
comments_seen: bool = false,
/// The mapping that bare/dotted `key = value` lines attach to. Starts at root,
/// repointed by each `[table]` header.
current_table: AST.Node.Id = 0,
/// Per-mapping provenance, used to enforce TOML's table/key conflict rules.
/// Absent ⇒ a value node or the root.
table_meta: std.AutoHashMapUnmanaged(AST.Node.Id, TableMeta) = .empty,
/// Error-recovery mode: when set, a failed top-level statement is recorded
/// into `diagnostics` and the parser resyncs to the next statement boundary
/// (see `resync`) instead of aborting. Off by default; `parseCollecting` turns
/// it on. `error.OutOfMemory` is never recovered (it is not a document
/// defect). Mirrors fig's `Parser.recover`.
recover: bool = false,
/// Every parse error hit so far, in source order. Populated on EVERY failure
/// (not just in `recover` mode) — the single-shot entry points
/// (`parse`/`parseWithReport`) still stop at the first, but the location is
/// captured the same way either way.
diagnostics: std.ArrayList(Diagnostic) = .empty,
/// Authoring-time lints collected during the parse (see `Warning.Code` — none
/// identified yet, so this always stays empty; kept for shape parity).
warnings: std.ArrayList(Warning) = .empty,
/// Overrides the next diagnostic's offset when the cursor has already scanned
/// past the token that actually offends (set via `failAt`/`failSpan`, cleared
/// after each statement) — `self.pos`'s current token is the right answer
/// everywhere else. Mirrors fig's `Parser.fail_offset`.
fail_offset: ?usize = null,
/// The offending span's end, paired with `fail_offset` (set via `failSpan`).
/// Mirrors fig's `Parser.fail_end`.
fail_end: ?usize = null,
/// How a table (mapping) node came to exist — determines whether a later header
/// or dotted key may target/extend it.
const TableMeta = struct {
/// Defined by its own `[header]` (or `[[array]]` element). Cannot be
/// redefined by a header, nor extended by dotted keys from another line.
explicit: bool = false,
/// Created as the value of a dotted-key segment (`a.b = 1` makes `a`).
dotted: bool = false,
/// Created only as an intermediate on a header path (`[a.b.c]` makes
/// `a`, `a.b`). May still be promoted to `explicit` by its own header.
implicit: bool = false,
/// A sequence node created by `[[array.of.tables]]`. Only such sequences
/// may be navigated into (last element) or extended; a static `= [...]`
/// array has no meta and is therefore closed.
aot: bool = false,
/// A mapping from an inline `{ ... }` table — fully defined and closed; no
/// header or dotted key may extend it.
inline_table: bool = false,
};
const KeySeg = struct { str: []const u8, span: Span };
pub const ParseError = error{
NotImplemented,
UnexpectedToken,
UnclosedString,
BadEscape,
InvalidUnicode,
InvalidNumber,
InvalidDatetime,
InvalidKey,
DuplicateKey,
TrailingContent,
InvalidUtf8,
};
pub const ParserError = ParseError || Tokenizer.TokenizeError || std.mem.Allocator.Error;
const parse_diagnostic = @import("../../parse_diagnostic.zig");
/// Every error this parser can produce — both the tokenizer (which runs to
/// completion before the statement loop starts, so its failures need
/// describing too) and the parser proper. Just an alias for `ParserError`
/// (which already unions both): named to match the sibling `describe`/
/// `shortLabel` pair in `languages/json/parser.zig` and
/// `languages/fig/parser.zig`.
pub const Error = ParserError;
/// The teaching message for `code` — one sentence naming the fix, same
/// contract as fig's `describe` (DESIGN.md "every diagnostic names the fix").
pub fn describe(code: Error) []const u8 {
return switch (code) {
error.NotImplemented => "this TOML construct is not implemented by this parser",
error.UnexpectedToken => "unexpected token here; check for a missing `=`, `.`, `,`, or closing `]`/`}`",
error.UnclosedString => "unclosed string; a single-line string cannot contain a literal newline — close the quote, or use a triple-quoted string (`\"\"\"`/`'''`) for multi-line text",
error.BadEscape => "invalid escape; basic strings support \\b \\t \\n \\f \\r \\\" \\\\ \\uXXXX \\UXXXXXXXX — use a literal string ('...') for raw text with backslashes",
error.InvalidUnicode => "invalid unicode escape; the hex digits do not form a valid Unicode codepoint",
error.InvalidNumber => "not a valid TOML number; check the radix prefix, digit grouping (a single `_` between digits, none leading/trailing), and that there is no leading zero",
error.InvalidDatetime => "not a valid RFC 3339 date/time",
error.InvalidKey => "invalid key; a bare key allows only letters, digits, `-`, and `_` — quote it for anything else",
error.DuplicateKey => "this key or table conflicts with one already defined; a TOML key or table may be defined only once",
error.TrailingContent => "unexpected content after this line's value; each TOML statement must end its line (a `#` comment needs whitespace before it)",
error.InvalidUtf8 => "this file is not valid UTF-8; TOML documents must be UTF-8 encoded",
error.UnexpectedCarriageReturn => "a bare `\\r` must be followed by `\\n`; TOML line endings are `\\n` or `\\r\\n`",
error.BadKey => "invalid key; expected a bare key, a quoted key, `.`, or `=` here",
error.BadValue => "not a recognized value; TOML values are strings, numbers, booleans, datetimes, arrays, or inline tables",
error.BadControlChar => "control characters are not allowed here (only tab is permitted outside a multi-line string)",
error.OutOfMemory => "out of memory",
};
}
/// A short (few-word) noun phrase for `code` — for a caret annotation
/// (`^ duplicate key/table`), same purpose as fig's `shortLabel`.
pub fn shortLabel(code: Error) []const u8 {
return switch (code) {
error.NotImplemented => "not implemented",
error.UnexpectedToken => "unexpected token",
error.UnclosedString => "unclosed string",
error.BadEscape => "invalid escape",
error.InvalidUnicode => "invalid unicode escape",
error.InvalidNumber => "invalid number",
error.InvalidDatetime => "invalid datetime",
error.InvalidKey => "invalid key",
error.DuplicateKey => "duplicate key/table",
error.TrailingContent => "trailing content",
error.InvalidUtf8 => "invalid UTF-8",
error.UnexpectedCarriageReturn => "bare CR",
error.BadKey => "invalid key",
error.BadValue => "invalid value",
error.BadControlChar => "control character",
error.OutOfMemory => "out of memory",
};
}
/// A parse failure plus the byte span where it fired. Most sites need no
/// special handling: the statement loop (`parseOnce`) anchors on whatever
/// token `self.pos` rests on when the error propagates back up to it, which
/// is correct for a plain token mismatch (the cursor hasn't advanced past the
/// bad token yet). Sites where the cursor HAS already scanned past the real
/// offender (a duplicate key/table noticed only after the rest of the line
/// parsed) call `failAt`/`failSpan` to pin the precise span instead — mirrors
/// fig's `Parser.Diagnostic` / `fail_offset`/`fail_end` exactly.
pub const Diagnostic = struct {
code: Error,
offset: usize,
end: ?usize = null,
/// 1-based line/column of `offset`, plus the full offending line.
pub fn locate(self: Diagnostic, source: []const u8) parse_diagnostic.Location {
return parse_diagnostic.locateOffset(source, self.offset);
}
/// Render `file:line:col: error: <message>` + source line + caret.
pub fn renderAlloc(self: Diagnostic, allocator: std.mem.Allocator, source: []const u8, file: []const u8) std.mem.Allocator.Error![]u8 {
return parse_diagnostic.renderReportAlloc(allocator, source, self.offset, file, "error", describe(self.code));
}
};
/// An authoring-time lint: the document is valid TOML, but the shape is a
/// likely mistake. TOML's grammar is stricter than fig/JSON's — no
/// literal-else-string sniffing ambiguity, and a duplicate key/table is
/// already a hard parse ERROR rather than something silently accepted that
/// would need a warning instead — so no lint has been identified yet; `Code`
/// is a real (empty) type rather than omitted, so `Report`'s shape matches
/// every other language's and the CLI's report-rendering glue
/// (`main.zig`'s `renderAll`) needs no per-language special case.
pub const Warning = struct {
code: Code,
offset: usize,
end: ?usize = null,
pub const Code = enum {};
pub fn describeWarning(code: Code) []const u8 {
// `Code` is uninhabited (see its doc comment) — no value can ever
// reach here.
_ = code;
unreachable;
}
pub fn shortLabel(code: Code) []const u8 {
_ = code;
unreachable;
}
pub fn locate(self: Warning, source: []const u8) parse_diagnostic.Location {
return parse_diagnostic.locateOffset(source, self.offset);
}
pub fn renderAlloc(self: Warning, allocator: std.mem.Allocator, source: []const u8, file: []const u8) std.mem.Allocator.Error![]u8 {
return parse_diagnostic.renderReportAlloc(allocator, source, self.offset, file, "warning", describeWarning(self.code));
}
};
/// Everything a parse reports besides the tree — mirrors fig's `Report`
/// (`languages/fig/parser.zig`) and JSON's twin field-for-field, so the CLI
/// treats every language's report the same shape (see `main.zig`'s
/// `checkOne`).
pub const Report = struct {
diag: ?Diagnostic = null,
/// Every parse error, in source order — populated ONLY by the recovering
/// entry point (`parseCollecting`), which resyncs past each failed
/// statement and keeps going. The single-shot `parse`/`parseWithReport`
/// stop at the first error and leave this empty, setting `diag` alone.
errors: []const Diagnostic = &.{},
warnings: []const Warning = &.{},
};
pub fn parse(allocator: std.mem.Allocator, input: []const u8, format: Type) ParserError!Document {
return parseImpl(allocator, input, format, null, false);
}
/// `parse`, but also fills `out`: `diag` on failure (error code + byte span,
/// for `file:line:col` teaching messages), `warnings` always (currently
/// always empty — see `Warning`'s doc comment). Mirrors fig's
/// `Parser.parseWithReport` — the hook the CLI renders reports from.
pub fn parseWithReport(allocator: std.mem.Allocator, input: []const u8, format: Type, out: *Report) ParserError!Document {
return parseImpl(allocator, input, format, out, false);
}
/// `parseWithReport`, but recovers past each failed top-level statement
/// (resyncing to the next statement boundary — see `resync`) to collect the
/// WHOLE file's diagnostics in one pass (`out.errors`, source order) rather
/// than stopping at the first. On any error the return value is still the
/// first error code and the tree is NOT built (both consumers discard it); a
/// clean parse returns the Document exactly as `parseWithReport` would.
/// `out.diag` mirrors `errors[0]`. Mirrors fig's `Parser.parseCollecting`.
pub fn parseCollecting(allocator: std.mem.Allocator, input: []const u8, format: Type, out: *Report) ParserError!Document {
return parseImpl(allocator, input, format, out, true);
}
fn parseImpl(allocator: std.mem.Allocator, input: []const u8, format: Type, out: ?*Report, recover: bool) ParserError!Document {
var parser: Parser = .{ .allocator = allocator, .recover = recover };
defer parser.diagnostics.deinit(allocator);
defer parser.warnings.deinit(allocator);
const result = parser.parseOnce(input, format);
// Warnings are duped out on every exit path (valid alongside a failure
// too), before the list above is freed.
if (out) |o| o.warnings = allocator.dupe(Warning, parser.warnings.items) catch &.{};
return result catch |err| {
if (out) |o| {
if (parser.diagnostics.items.len > 0) {
o.diag = parser.diagnostics.items[0];
if (recover) o.errors = allocator.dupe(Diagnostic, parser.diagnostics.items) catch &.{};
}
}
parser.nodes.deinit(allocator);
parser.spans.deinit(allocator);
for (parser.owned_strings.items) |s| allocator.free(s);
parser.owned_strings.deinit(allocator);
return err;
};
}
pub fn parseAbstract(allocator: std.mem.Allocator, input: []const u8, format: Type) ParserError!AST {
const doc = try parse(allocator, input, format);
allocator.free(doc.node_spans);
return doc.ast;
}
/// One top-level statement (`key = value` / `[table]` / `[[array.of.tables]]`)
/// plus the line-end check that must follow it — extracted from `parseOnce`'s
/// main loop so it can be `catch`-wrapped there (single-shot: propagate;
/// `recover`: record + resync — see `resync`) without duplicating the switch.
fn dispatchStatement(self: *Parser) ParserError!void {
switch (self.peek().kind) {
.key => try self.parseKeyValue(),
.open_bracket => try self.parseTableHeader(),
.double_open_bracket => try self.parseArrayTable(),
else => return error.UnexpectedToken,
}
try self.requireLineEnd();
}
/// After a parse error while parsing one top-level statement (only reachable
/// in `recover` mode), scan forward from `start` for the next `.newline`/
/// `.end_of_file` OUTSIDE any bracket nesting — tracking `[`/`[[`/`{` opens
/// and `]`/`]]`/`}` closes so a broken multi-line array/inline table's
/// interior newlines don't count as a safe statement boundary (a stray
/// unmatched closer just clamps depth at 0 instead of going negative, rather
/// than being treated as a real close of anything — unlike JSON's recovery,
/// nothing here is actually CONSUMED as a container close, so there is no
/// "does this closer match what's open" question to get wrong). Always finds
/// one (`end_of_file` is always the tokenizer's last token), so this
/// terminates. The returned token is not consumed by this function — the
/// caller's `skipBlank` does that for a newline, or `atEnd()` ends the loop
/// for EOF, either way guaranteeing progress even when `start` already IS the
/// landing token.
fn resync(self: *Parser, start: usize) usize {
var depth: isize = 0;
var j = start;
while (j < self.tokens.len) : (j += 1) {
switch (self.tokens[j].kind) {
.open_bracket, .open_brace, .double_open_bracket => depth += 1,
.close_bracket, .close_brace, .double_close_bracket => {
if (depth > 0) depth -= 1;
},
.newline => if (depth == 0) return j,
.end_of_file => return j,
else => {},
}
}
return self.tokens.len - 1;
}
/// `resync`, then re-tokenizes everything from the landing point on and swaps
/// it in for the corresponding suffix of `self.tokens`, resetting `self.pos`
/// to the start of the fresh segment. Necessary because the tokenizer runs
/// ONCE over the whole file up front and is context-sensitive (it tracks
/// `[`/`{` nesting and a key-vs-value mode to disambiguate a bare key from a
/// bare date/number — see `Tokenizer.flow`/`in_value`): a genuinely unclosed
/// `[`/`{` leaves that context stuck in "inside a collection" for the REST OF
/// THE FILE, so every later statement — even syntactically perfect ones —
/// gets mistokenized and cascades into bogus errors. Re-tokenizing from the
/// resync point re-establishes the tokenizer's context from a known-clean
/// slate (a real top-level statement boundary is always `in_value = false`,
/// `flow` empty, by TOML's own grammar), which a purely parser-side resync
/// (adjusting only `self.pos`) cannot do — the corrupted token DATA is already
/// baked into `self.tokens` regardless of where `self.pos` points.
///
/// Returns `false` when the remainder can't even be lexed (a NEW tokenizer
/// failure on the fresh slice) — recorded as one final diagnostic, then the
/// caller stops recovering, same as landing on `end_of_file` normally.
fn resyncAndRetokenize(self: *Parser) ParserError!bool {
const sync = self.resync(self.pos);
const restart_byte = self.tokens[sync].span.start;
var tokenizer: Tokenizer = .{ .allocator = self.allocator, .str = self.source[restart_byte..], .version = self.version };
const fresh = tokenizer.tokenize() catch |err| {
try self.diagnostics.append(self.allocator, .{ .code = err, .offset = restart_byte + tokenizer.i });
return false;
};
// Spans from `tokenizer` are relative to the subslice — shift them back
// to absolute so they still index into `self.source` correctly (the AST
// spans built from here on, and any later diagnostic, depend on this).
for (fresh) |*t| {
t.span.start += restart_byte;
t.span.end += restart_byte;
}
self.allocator.free(self.tokens);
self.tokens = fresh;
self.pos = 0;
return true;
}
/// Return `err` with the diagnostic caret pinned to `offset` — for sites where
/// the cursor has already scanned past the token that actually offends.
/// Mirrors fig's `Parser.failAt`.
fn failAt(self: *Parser, offset: usize, err: Error) Error {
self.fail_offset = offset;
return err;
}
/// Like `failAt`, but also pins the offending token's end (`start..end`) so
/// the diagnostic carries a tight range. Mirrors fig's `Parser.failSpan`.
fn failSpan(self: *Parser, start: usize, end: usize, err: Error) Error {
self.fail_offset = start;
self.fail_end = end;
return err;
}
fn parseOnce(self: *Parser, input: []const u8, format: Type) ParserError!Document {
self.version = format;
self.source = input;
// A TOML file must be valid UTF-8 (catches all bad-utf8 fixtures at once).
if (!std.unicode.utf8ValidateSlice(input)) return error.InvalidUtf8;
var tokenizer: Tokenizer = .{ .allocator = self.allocator, .str = input, .version = format };
self.tokens = tokenizer.tokenize() catch |err| {
// The tokenizer's cursor sits on (or just after) the offending byte —
// precise enough for `file:line:col` without threading a location
// through its many scan-error sites (mirrors the parser's own
// token-span anchor below). Never recoverable: a lexical failure
// happens before the statement loop even starts.
try self.diagnostics.append(self.allocator, .{ .code = err, .offset = tokenizer.i });
return err;
};
defer self.allocator.free(self.tokens);
defer self.table_meta.deinit(self.allocator);
// Reserve so trivia-skipping can buffer leading comments without failing.
try self.pending_leading.ensureTotalCapacity(self.allocator, self.tokens.len);
defer self.pending_leading.deinit(self.allocator);
// On success the table is moved into the AST and this list emptied; on any
// error path it (and any owned comment slices) are freed here.
defer {
for (self.node_comments.items) |nc| {
self.allocator.free(nc.leading);
self.allocator.free(nc.dangling);
}
self.node_comments.deinit(self.allocator);
}
// Root mapping is node 0.
const root_id = try self.addNode(.{ .mapping = null }, Span.init(0, input.len));
self.current_table = root_id;
self.skipBlank();
while (!self.atEnd()) {
self.dispatchStatement() catch |err| {
if (err == error.OutOfMemory) return err; // never a document defect
// The cursor rests on the offending token by default (most sites
// return before consuming it); `failAt`/`failSpan` sites override
// this with a more precise position, then it's cleared so it
// can't leak into a LATER error that didn't set one.
const offset = self.fail_offset orelse self.peek().span.start;
const end = self.fail_end;
self.fail_offset = null;
self.fail_end = null;
try self.diagnostics.append(self.allocator, .{ .code = err, .offset = offset, .end = end });
if (!self.recover) return err;
if (!try self.resyncAndRetokenize()) break;
};
self.skipBlank();
}
// End-of-file orphan comments dangle off the table they sit in (the last one
// `current_table` points at). Mid-file orphans are instead claimed as the
// next key's / header's leading.
try self.claimDangling(self.current_table);
if (self.diagnostics.items.len > 0) {
// Recover mode collected 1+ errors (or a non-recover error that
// reached here would already have returned above — this only fires
// in recover mode): the tree may be malformed (dangling
// half-built tables), so don't try to finish building it — return
// the first failure's code, matching the non-recovering contract
// exactly. `parseImpl` reads the full list out of `self.diagnostics`
// for `Report.errors` before this unwinds.
return self.diagnostics.items[0].code;
}
const nodes = try self.nodes.toOwnedSlice(self.allocator);
errdefer self.allocator.free(nodes);
const spans = try self.spans.toOwnedSlice(self.allocator);
errdefer self.allocator.free(spans);
const owned = try self.owned_strings.toOwnedSlice(self.allocator);
var ast: AST = .{
.allocator = self.allocator,
.root = root_id,
.nodes = nodes,
.owned_strings = owned,
};
if (self.comments_seen) {
ast.node_comments = try self.node_comments.toOwnedSlice(self.allocator);
self.node_comments = .empty;
}
return .{
.source = input,
.ast = ast,
.node_spans = spans,
};
}
fn addNode(self: *Parser, kind: AST.Node.Kind, span: Span) ParserError!AST.Node.Id {
const id: AST.Node.Id = @intCast(self.nodes.items.len);
try self.nodes.append(self.allocator, .{ .id = id, .kind = kind });
try self.spans.append(self.allocator, span);
try self.node_comments.append(self.allocator, .{});
return id;
}
// ── Token cursor ────────────────────────────────────────────────────────────
fn peek(self: *Parser) Token {
return self.tokens[self.pos];
}
fn advance(self: *Parser) Token {
const t = self.tokens[self.pos];
if (self.pos + 1 < self.tokens.len) self.pos += 1;
return t;
}
fn atEnd(self: *Parser) bool {
return self.peek().kind == .end_of_file;
}
/// Skip whitespace and comments (but not newlines). A comment here is on the
/// current line, so it can trail a just-parsed value.
fn skipInline(self: *Parser) void {
while (true) switch (self.peek().kind) {
.whitespace => self.pos += 1,
.comment => {
self.captureComment(self.peek());
self.pos += 1;
},
else => return,
};
}
/// Skip whitespace, comments, and blank lines (newlines). A newline closes the
/// trailing-comment window, so comments past it lead the next entry.
fn skipBlank(self: *Parser) void {
while (true) switch (self.peek().kind) {
.whitespace => self.pos += 1,
.comment => {
self.captureComment(self.peek());
self.pos += 1;
},
.newline => {
self.last_value_id = null;
self.pos += 1;
},
else => return,
};
}
// ── Comments ─────────────────────────────────────────────────────────────────
/// Classify a comment token: trailing the most recent value when its window is
/// open (`last_value_id` set, no newline since), else buffered as leading.
fn captureComment(self: *Parser, tok: Token) void {
const c: AST.Comment = .{ .text = commentText(self.tokenText(tok)), .style = .line };
if (self.last_value_id) |id| {
self.node_comments.items[id].trailing = c;
self.comments_seen = true;
self.last_value_id = null; // one trailing per value
} else {
self.pending_leading.appendAssumeCapacity(c); // capacity reserved in parseOnce
}
}
/// Strip the leading `#` and surrounding spaces from a comment token's bytes
/// (which borrow `source`).
fn commentText(raw: []const u8) []const u8 {
const body = if (raw.len > 0 and raw[0] == '#') raw[1..] else raw;
return std.mem.trim(u8, body, " \t\r");
}
/// Hand the buffered leading comments to node `id` as an owned slice, then clear
/// the buffer (retaining its reserved capacity). No-op when nothing is buffered.
fn claimLeading(self: *Parser, id: AST.Node.Id) ParserError!void {
if (self.pending_leading.items.len == 0) return;
const owned = try self.allocator.dupe(AST.Comment, self.pending_leading.items);
self.pending_leading.clearRetainingCapacity();
self.node_comments.items[id].leading = owned;
self.comments_seen = true;
}
/// Hand buffered orphan comments (no key/header followed them — at end of file)
/// to table `id` as its `dangling` run. Mid-file orphans instead lead the next
/// key/header, so this only fires at EOF.
fn claimDangling(self: *Parser, id: AST.Node.Id) ParserError!void {
if (self.pending_leading.items.len == 0) return;
const owned = try self.allocator.dupe(AST.Comment, self.pending_leading.items);
self.pending_leading.clearRetainingCapacity();
self.node_comments.items[id].dangling = owned;
self.comments_seen = true;
}
/// After a statement, only trivia then a newline or EOF may follow.
fn requireLineEnd(self: *Parser) ParserError!void {
self.skipInline();
switch (self.peek().kind) {
.newline, .end_of_file => {},
else => return error.TrailingContent,
}
}
fn tokenText(self: *Parser, tok: Token) []const u8 {
return self.source[tok.span.start..tok.span.end];
}
// ── Statements ──────────────────────────────────────────────────────────────
/// Parse `[table.path]` and repoint `current_table`. The path is resolved from
/// the root; intermediates may pass through any existing table, the final must
/// be new (→ explicit) or an implicit path-table (→ promoted to explicit).
fn parseTableHeader(self: *Parser) ParserError!void {
_ = self.advance(); // '['
self.skipInline();
var segs: std.ArrayList(KeySeg) = .empty;
defer segs.deinit(self.allocator);
try self.parseKeyPath(&segs);
self.skipInline();
if (self.peek().kind != .close_bracket) return error.UnexpectedToken;
_ = self.advance();
if (segs.items.len == 0) return error.InvalidKey;
const cur = try self.navigateHeaderPath(0, segs.items[0 .. segs.items.len - 1]);
const final = segs.items[segs.items.len - 1];
if (self.lookupChild(cur, final.str)) |child| {
// The cursor has already consumed the whole `[header]` line by this
// point, so the default "current token" anchor would land on the NEXT
// line's content — pin it to the conflicting final path segment
// instead.
if (self.nodes.items[child].kind != .mapping) return self.failSpan(final.span.start, final.span.end, error.DuplicateKey);
const m = self.table_meta.get(child) orelse TableMeta{};
if (m.explicit or m.dotted or m.inline_table) return self.failSpan(final.span.start, final.span.end, error.DuplicateKey);
try self.table_meta.put(self.allocator, child, .{ .explicit = true });
self.current_table = child;
} else {
self.current_table = try self.createTable(cur, final, .{ .explicit = true });
}
}
/// Parse `[[array.of.tables]]`: navigate the path from root (intermediates like
/// a header), then create-or-extend the final array-of-tables, appending a fresh
/// element table that becomes `current_table`.
fn parseArrayTable(self: *Parser) ParserError!void {
_ = self.advance(); // '[['
self.skipInline();
var segs: std.ArrayList(KeySeg) = .empty;
defer segs.deinit(self.allocator);
try self.parseKeyPath(&segs);
self.skipInline();
if (self.peek().kind != .double_close_bracket) return error.UnexpectedToken;
_ = self.advance();
if (segs.items.len == 0) return error.InvalidKey;
const cur = try self.navigateHeaderPath(0, segs.items[0 .. segs.items.len - 1]);
const final = segs.items[segs.items.len - 1];
if (self.lookupChild(cur, final.str)) |child| {
const m = self.table_meta.get(child) orelse TableMeta{};
// See `parseTableHeader`: the `[[header]]` line is fully consumed
// already, so pin the caret to the conflicting final segment.
if (self.nodes.items[child].kind != .sequence or !m.aot) return self.failSpan(final.span.start, final.span.end, error.DuplicateKey);
self.current_table = try self.appendArrayElement(child);
} else {
const seq_id = try self.addNode(.{ .sequence = null }, final.span);
try self.appendKeyValue(cur, final, seq_id);
try self.table_meta.put(self.allocator, seq_id, .{ .aot = true });
self.current_table = try self.appendArrayElement(seq_id);
}
}
/// Walk a header/array-of-tables path of intermediate segments from `start`,
/// creating missing tables (implicit) and descending into existing ones. An
/// array-of-tables intermediate descends into its last element.
fn navigateHeaderPath(self: *Parser, start: AST.Node.Id, intermediates: []const KeySeg) ParserError!AST.Node.Id {
var cur = start;
for (intermediates) |seg| {
if (self.lookupChild(cur, seg.str)) |child| {
// `descend`'s only failures are conflicts on THIS intermediate
// segment, but by the time they propagate here the whole header
// line is already consumed — pin the caret to `seg` instead of
// whatever the default (current token) would land on.
cur = self.descend(child) catch |err| return self.failSpan(seg.span.start, seg.span.end, err);
} else {
cur = try self.createTable(cur, seg, .{ .implicit = true });
}
}
return cur;
}
/// Resolve an existing path node to the mapping to continue from: a plain table
/// directly, an array-of-tables to its last element. A non-table value, a
/// static array, or a closed inline table is an error.
fn descend(self: *Parser, child: AST.Node.Id) ParserError!AST.Node.Id {
return switch (self.nodes.items[child].kind) {
.mapping => blk: {
const m = self.table_meta.get(child) orelse TableMeta{};
if (m.inline_table) return error.DuplicateKey;
break :blk child;
},
.sequence => blk: {
const m = self.table_meta.get(child) orelse TableMeta{};
if (!m.aot) return error.DuplicateKey;
break :blk try self.lastElement(child);
},
else => error.DuplicateKey,
};
}
fn lastElement(self: *Parser, seq_id: AST.Node.Id) ParserError!AST.Node.Id {
var last = self.nodes.items[seq_id].kind.sequence orelse return error.DuplicateKey;
while (self.nodes.items[last].next_sibling) |n| last = n;
return last;
}
fn appendArrayElement(self: *Parser, seq_id: AST.Node.Id) ParserError!AST.Node.Id {
const elem = try self.addNode(.{ .mapping = null }, self.spans.items[seq_id]);
if (self.nodes.items[seq_id].kind.sequence) |first| {
var last = first;
while (self.nodes.items[last].next_sibling) |n| last = n;
self.nodes.items[last].next_sibling = elem;
} else {
self.nodes.items[seq_id].kind = .{ .sequence = elem };
}
return elem;
}
/// Parse a `key = value` line (the key possibly dotted), attaching to
/// `current_table`. Dotted intermediates create/extend dotted tables but may
/// not descend into an explicitly-defined table (TOML forbids using dotted keys
/// to append to a `[table]`) nor through a non-table value.
fn parseKeyValue(self: *Parser) ParserError!void {
var segs: std.ArrayList(KeySeg) = .empty;
defer segs.deinit(self.allocator);
try self.parseKeyPath(&segs);
self.skipInline();
if (self.peek().kind != .equals) return error.UnexpectedToken;
_ = self.advance(); // '='
self.skipInline();
const cur = try self.navigateDottedPath(self.current_table, segs.items[0 .. segs.items.len - 1]);
const final = segs.items[segs.items.len - 1];
// The cursor already sits on the VALUE token here (past `=`) — pin the
// caret to the conflicting key segment instead of the value.
if (self.lookupChild(cur, final.str) != null) return self.failSpan(final.span.start, final.span.end, error.DuplicateKey);
const value_id = try self.parseValue();
// The value is now the trailing-comment candidate for the rest of this line
// (a `# comment` after it, captured by the upcoming `requireLineEnd`).
self.last_value_id = value_id;
try self.appendKeyValue(cur, final, value_id);
}
/// Walk dotted-key intermediates from `start`, creating missing tables (dotted)
/// and descending into dotted/implicit ones. Descending into an explicitly
/// defined table, a closed inline table, or a non-table value is an error —
/// dotted keys may not append to a `[table]`.
fn navigateDottedPath(self: *Parser, start: AST.Node.Id, intermediates: []const KeySeg) ParserError!AST.Node.Id {
var cur = start;
for (intermediates) |seg| {
if (self.lookupChild(cur, seg.str)) |child| {
// The cursor is already past this whole dotted key (and its `=`)
// by the time either check below can fire — pin the caret to the
// conflicting intermediate segment itself.
if (self.nodes.items[child].kind != .mapping) return self.failSpan(seg.span.start, seg.span.end, error.DuplicateKey);
const m = self.table_meta.get(child) orelse TableMeta{};
if (m.explicit or m.inline_table) return self.failSpan(seg.span.start, seg.span.end, error.DuplicateKey);
cur = child;
} else {
cur = try self.createTable(cur, seg, .{ .dotted = true });
}
}
return cur;
}
/// Parse a dotted key path (`a.b.c`); cursor must be at the first `.key`. Stops
/// when the next significant token is not a dot.
fn parseKeyPath(self: *Parser, segs: *std.ArrayList(KeySeg)) ParserError!void {
while (true) {
const tok = self.peek();
if (tok.kind != .key) return error.UnexpectedToken;
_ = self.advance();
try segs.append(self.allocator, .{ .str = try self.decodeKey(tok), .span = tok.span });
self.skipInline();
if (self.peek().kind != .dot) return;
_ = self.advance();
self.skipInline();
}
}
/// Value node id of a child of `map_id` keyed `key`, or null.
fn lookupChild(self: *Parser, map_id: AST.Node.Id, key: []const u8) ?AST.Node.Id {
var cur = self.nodes.items[map_id].kind.mapping;
while (cur) |id| : (cur = self.nodes.items[id].next_sibling) {
const kv = self.nodes.items[id].kind.keyvalue;
if (std.mem.eql(u8, self.nodes.items[kv.key].kind.string, key)) return kv.value;
}
return null;
}
/// Append a `key = value` entry to mapping `map_id`.
fn appendKeyValue(self: *Parser, map_id: AST.Node.Id, key: KeySeg, value_id: AST.Node.Id) ParserError!void {
const key_id = try self.addNode(.{ .string = key.str }, key.span);
// A leading comment block above this line (or above a `[header]`, since
// headers also route through here) binds to the key node.
try self.claimLeading(key_id);
const value_end = self.spans.items[value_id].end;
const kv_id = try self.addNode(
.{ .keyvalue = .{ .key = key_id, .value = value_id } },
Span.init(key.span.start, value_end),
);
if (self.nodes.items[map_id].kind.mapping) |first| {
var last = first;
while (self.nodes.items[last].next_sibling) |n| last = n;
self.nodes.items[last].next_sibling = kv_id;
} else {
self.nodes.items[map_id].kind = .{ .mapping = kv_id };
}
}
/// Create an empty child table under `parent` keyed `key`, record its origin,
/// and return the new mapping node id.
fn createTable(self: *Parser, parent: AST.Node.Id, key: KeySeg, meta: TableMeta) ParserError!AST.Node.Id {
const map_id = try self.addNode(.{ .mapping = null }, key.span);
try self.appendKeyValue(parent, key, map_id);
try self.table_meta.put(self.allocator, map_id, meta);
return map_id;
}
fn parseValue(self: *Parser) ParserError!AST.Node.Id {
const tok = self.peek();
switch (tok.kind) {
.string => {
_ = self.advance();
const decoded = try self.decodeString(self.tokenText(tok));
return self.addNode(.{ .string = decoded }, tok.span);
},
.number => {
_ = self.advance();
const raw = self.tokenText(tok);
const kind = try classifyNumber(raw);
// Store the value in canonical, format-independent form (decimal,
// no underscores) so TOML→JSON/YAML conversion is direct. The
// original source text is still recoverable via node_spans, so a
// future round-trip editor loses nothing.
const canon = switch (kind) {
.integer => try self.canonicalInt(raw),
.float => try self.canonicalFloat(raw),
};
return self.addNode(.{ .number = .{ .raw = canon, .kind = kind } }, tok.span);
},
.datetime => {
_ = self.advance();
const raw = self.tokenText(tok);
const shape = try self.classifyDatetime(raw);
return self.addNode(.{ .extended = .{ .text = raw, .kind = shape } }, tok.span);
},
.boolean => {
_ = self.advance();
return self.addNode(.{ .boolean = self.tokenText(tok)[0] == 't' }, tok.span);
},
.open_bracket => return self.parseArray(),
.open_brace => return self.parseInlineTable(),
else => return error.UnexpectedToken,
}
}
/// Parse a `[ value, value, ... ]` array. Arrays may span lines and carry a
/// trailing comma; elements are heterogeneous.
fn parseArray(self: *Parser) ParserError!AST.Node.Id {
const start = self.peek().span.start;
_ = self.advance(); // '['
const seq_id = try self.addNode(.{ .sequence = null }, Span.init(start, start + 1));
var last: ?AST.Node.Id = null;
while (true) {
self.skipBlankFlow();
if (self.peek().kind == .close_bracket) break;
const elem = try self.parseValue();
if (last) |l| {
self.nodes.items[l].next_sibling = elem;
} else {
self.nodes.items[seq_id].kind = .{ .sequence = elem };
}
last = elem;
self.skipBlankFlow();
switch (self.peek().kind) {
.comma => _ = self.advance(),
.close_bracket => break,
else => return error.UnexpectedToken,
}
}
const end = self.peek().span.end;
_ = self.advance(); // ']'
self.spans.items[seq_id] = Span.init(start, end);
return seq_id;
}
/// Parse a `{ key = value, ... }` inline table. TOML 1.0 inline tables are
/// single-line (no newlines inside) and have no trailing comma.
fn parseInlineTable(self: *Parser) ParserError!AST.Node.Id {
// TOML 1.1 permits newlines and a trailing comma inside inline tables; 1.0
// requires everything on one line with no trailing comma.
const allow_nl = self.version == .TOML_1_1;
const start = self.peek().span.start;
_ = self.advance(); // '{'
const map_id = try self.addNode(.{ .mapping = null }, Span.init(start, start + 1));
try self.table_meta.put(self.allocator, map_id, .{ .inline_table = true });
self.skipFlowWs(allow_nl);
if (self.peek().kind != .close_brace) {
while (true) {
try self.parseInlineEntry(map_id);
self.skipFlowWs(allow_nl);
switch (self.peek().kind) {
.comma => {
_ = self.advance();
self.skipFlowWs(allow_nl);
if (self.peek().kind == .close_brace) {
if (!allow_nl) return error.UnexpectedToken; // trailing comma (1.0)
break;
}
},
.close_brace => break,
else => return error.UnexpectedToken,
}
}
}
const end = self.peek().span.end;
if (self.peek().kind != .close_brace) return error.UnexpectedToken;
_ = self.advance(); // '}'
self.spans.items[map_id] = Span.init(start, end);
return map_id;
}
/// One `key = value` (key possibly dotted) inside an inline table.
fn parseInlineEntry(self: *Parser, table_id: AST.Node.Id) ParserError!void {
const allow_nl = self.version == .TOML_1_1;
var segs: std.ArrayList(KeySeg) = .empty;
defer segs.deinit(self.allocator);
// Inline-table keys are lexed in value mode, so a bare key arrives as a
// .number/.boolean/.datetime token; reinterpret it by position.
while (true) {
const tok = self.peek();
const key = try self.decodeInlineKey(tok);
_ = self.advance();
try segs.append(self.allocator, .{ .str = key, .span = tok.span });
self.skipFlowWs(allow_nl);
if (self.peek().kind != .dot) break;
_ = self.advance();
self.skipFlowWs(allow_nl);
}
if (self.peek().kind != .equals) return error.UnexpectedToken;
_ = self.advance();
self.skipFlowWs(allow_nl);
const cur = try self.navigateDottedPath(table_id, segs.items[0 .. segs.items.len - 1]);
const final = segs.items[segs.items.len - 1];
// See `parseKeyValue`: the cursor is past `=` here — pin the caret to the
// conflicting key segment instead of the value.
if (self.lookupChild(cur, final.str) != null) return self.failSpan(final.span.start, final.span.end, error.DuplicateKey);
const value_id = try self.parseValue();
try self.appendKeyValue(cur, final, value_id);
}
fn decodeInlineKey(self: *Parser, tok: Token) ParserError![]const u8 {
return switch (tok.kind) {
.string => self.decodeString(self.tokenText(tok)), // quoted key
.key => self.tokenText(tok),
.number, .boolean, .datetime => blk: {
const text = self.tokenText(tok);
if (text.len == 0) return error.InvalidKey;
for (text) |c| if (!Tokenizer.isBareKeyChar(c)) return error.InvalidKey;
break :blk text;
},
else => error.UnexpectedToken,
};
}
/// Skip whitespace, comments, and newlines — for inside a multi-line array.
fn skipBlankFlow(self: *Parser) void {
self.skipBlank();
}
/// Skip inline-table separators: whitespace + comments, plus newlines when the
/// version permits them inside inline tables (1.1).
fn skipFlowWs(self: *Parser, allow_nl: bool) void {
if (allow_nl) self.skipBlank() else self.skipInline();
}
// ── Key / string decoding ───────────────────────────────────────────────────
fn decodeKey(self: *Parser, tok: Token) ParserError![]const u8 {
const raw = self.tokenText(tok);
if (raw.len == 0) return error.InvalidKey;
return switch (raw[0]) {
'"', '\'' => self.decodeString(raw),
else => raw, // bare key
};
}
/// Decode any of the four TOML string forms to its byte value. Returns a slice
/// of `source` when no transformation is needed, else an owned (freed via the
/// AST) allocation.
fn decodeString(self: *Parser, raw: []const u8) ParserError![]const u8 {
if (raw.len < 2) return error.UnclosedString;
const q = raw[0];
const triple = raw.len >= 6 and raw[1] == q and raw[2] == q;
if (q == '\'') {
// Literal: no escapes.
if (triple) {
var inner = raw[3 .. raw.len - 3];
inner = trimLeadingNewline(inner);
return inner;
}
return raw[1 .. raw.len - 1];
}
// Basic (q == '"').
if (triple) {
const inner = trimLeadingNewline(raw[3 .. raw.len - 3]);
return self.decodeBasic(inner, true);
}
const inner = raw[1 .. raw.len - 1];
if (std.mem.indexOfScalar(u8, inner, '\\') == null) return inner;
return self.decodeBasic(inner, false);
}
/// A multi-line string with a newline immediately after the opening delimiter
/// drops that first newline.
fn trimLeadingNewline(inner: []const u8) []const u8 {
if (inner.len >= 1 and inner[0] == '\n') return inner[1..];
if (inner.len >= 2 and inner[0] == '\r' and inner[1] == '\n') return inner[2..];
return inner;
}
fn decodeBasic(self: *Parser, inner: []const u8, multiline: bool) ParserError![]const u8 {
var out: std.ArrayList(u8) = .empty;
errdefer out.deinit(self.allocator);
var i: usize = 0;
while (i < inner.len) {
const c = inner[i];
if (c != '\\') {
try out.append(self.allocator, c);
i += 1;
continue;
}
if (i + 1 >= inner.len) return error.BadEscape;
const n = inner[i + 1];
switch (n) {
'b' => try out.append(self.allocator, 0x08),
't' => try out.append(self.allocator, '\t'),
'n' => try out.append(self.allocator, '\n'),
'f' => try out.append(self.allocator, 0x0c),
'r' => try out.append(self.allocator, '\r'),
'"' => try out.append(self.allocator, '"'),
'\\' => try out.append(self.allocator, '\\'),
'u' => i = try self.appendUnicode(&out, inner, i + 2, 4) - 2,
'U' => i = try self.appendUnicode(&out, inner, i + 2, 8) - 2,
// TOML 1.1: \e is ESC (U+001B); \xHH is shorthand for \u00HH.
'e' => {
if (self.version == .TOML_1_0) return error.BadEscape;
try out.append(self.allocator, 0x1b);
},
'x' => {
if (self.version == .TOML_1_0) return error.BadEscape;
i = try self.appendUnicode(&out, inner, i + 2, 2) - 2;
},
' ', '\t', '\n', '\r' => {
if (!multiline) return error.BadEscape;
// Line-ending backslash: `\` + optional whitespace + newline
// trims all following whitespace up to the next content.
var j = i + 1;
while (j < inner.len and (inner[j] == ' ' or inner[j] == '\t')) j += 1;
if (j >= inner.len or (inner[j] != '\n' and inner[j] != '\r')) return error.BadEscape;
while (j < inner.len and (inner[j] == ' ' or inner[j] == '\t' or inner[j] == '\n' or inner[j] == '\r')) j += 1;
i = j;
continue;
},
else => return error.BadEscape,
}
i += 2;
}
const slice = try out.toOwnedSlice(self.allocator);
errdefer self.allocator.free(slice);
try self.owned_strings.append(self.allocator, slice);
return slice;
}
/// Decode `n` hex digits at `inner[at..]` into a UTF-8 codepoint appended to
/// `out`; returns the index just past the digits.
fn appendUnicode(self: *Parser, out: *std.ArrayList(u8), inner: []const u8, at: usize, n: usize) ParserError!usize {
if (at + n > inner.len) return error.BadEscape;
const cp = std.fmt.parseInt(u21, inner[at .. at + n], 16) catch return error.InvalidUnicode;
Unicode.encodeAppend(out, self.allocator, cp) catch |err| switch (err) {
error.InvalidCodepoint => return error.InvalidUnicode,
error.OutOfMemory => return error.OutOfMemory,
};
return at + n;
}
// ── Number validation / classification ──────────────────────────────────────
const NumberKind = @FieldType(AST.Node.Kind.Number, "kind");
fn classifyNumber(raw: []const u8) ParserError!NumberKind {
if (raw.len == 0) return error.InvalidNumber;
// Special floats.
if (eqAny(raw, &.{ "inf", "+inf", "-inf", "nan", "+nan", "-nan" })) return .float;
// Radix-prefixed integers (no sign permitted).
if (raw.len >= 2 and raw[0] == '0') switch (raw[1]) {
'x' => return if (validUnderscored(raw[2..], ascii.isHex)) .integer else error.InvalidNumber,
'o' => return if (validUnderscored(raw[2..], ascii.isOctal)) .integer else error.InvalidNumber,
'b' => return if (validUnderscored(raw[2..], ascii.isBinary)) .integer else error.InvalidNumber,
else => {},
};
var body = raw;
if (body[0] == '+' or body[0] == '-') body = body[1..];
if (body.len == 0) return error.InvalidNumber;
// Split mantissa / exponent.
var mantissa = body;
var exponent: ?[]const u8 = null;
if (std.mem.indexOfAny(u8, body, "eE")) |e| {
mantissa = body[0..e];
exponent = body[e + 1 ..];
}
// Mantissa: int part, optional `.fraction`.
var int_part = mantissa;
var frac_part: ?[]const u8 = null;
if (std.mem.indexOfScalar(u8, mantissa, '.')) |d| {
int_part = mantissa[0..d];
frac_part = mantissa[d + 1 ..];
}
if (!validDecimalInt(int_part)) return error.InvalidNumber;
var is_float = false;
if (frac_part) |f| {
if (!validUnderscored(f, ascii.isDigit)) return error.InvalidNumber;
is_float = true;
}
if (exponent) |e| {
var exp = e;
if (exp.len > 0 and (exp[0] == '+' or exp[0] == '-')) exp = exp[1..];
if (!validUnderscored(exp, ascii.isDigit)) return error.InvalidNumber;
is_float = true;
}
return if (is_float) .float else .integer;
}
/// A decimal integer literal: `0`, or a non-zero-leading run of digits. Used for
/// the standalone integer and for a float's integer part (leading zeros banned
/// in both: `01` and `03.14` are invalid).
fn validDecimalInt(s: []const u8) bool {
if (!validUnderscored(s, ascii.isDigit)) return false;
if (s.len > 1 and s[0] == '0') return false; // no leading zeros
return true;
}
/// Non-empty, all chars satisfy `pred` or are `_`, and every `_` sits between
/// two `pred` digits (no leading/trailing/doubled underscore).
fn validUnderscored(s: []const u8, comptime pred: fn (u8) bool) bool {
if (s.len == 0) return false;
if (s[0] == '_' or s[s.len - 1] == '_') return false;
var prev_us = false;
for (s) |c| {
if (c == '_') {
if (prev_us) return false;
prev_us = true;
} else if (pred(c)) {
prev_us = false;
} else return false;
}
return true;
}
/// Canonicalize an integer literal (any radix, underscores, sign) to a decimal
/// string. Returns `raw` unchanged when already canonical, else an owned copy.
fn canonicalInt(self: *Parser, raw: []const u8) ParserError![]const u8 {
var buf: [80]u8 = undefined;
var n: usize = 0;
for (raw) |c| {
if (c == '_') continue;
if (n >= buf.len) return error.InvalidNumber;
buf[n] = c;
n += 1;
}
const v = std.fmt.parseInt(i64, buf[0..n], 0) catch return error.InvalidNumber;
var out: [24]u8 = undefined;
const s = std.fmt.bufPrint(&out, "{d}", .{v}) catch return error.InvalidNumber;
if (std.mem.eql(u8, s, raw)) return raw;
return self.intern(s);
}
/// Canonicalize a float literal: special values to `inf`/`-inf`/`nan`, and strip
/// digit-group underscores (the remaining decimal/exponent form is valid JSON).
fn canonicalFloat(self: *Parser, raw: []const u8) ParserError![]const u8 {
if (eqAny(raw, &.{ "inf", "+inf" })) return "inf";
if (std.mem.eql(u8, raw, "-inf")) return "-inf";
if (eqAny(raw, &.{ "nan", "+nan", "-nan" })) return "nan";
if (std.mem.indexOfScalar(u8, raw, '_') == null) return raw;
var buf: [80]u8 = undefined;
var n: usize = 0;
for (raw) |c| {
if (c == '_') continue;
if (n >= buf.len) return error.InvalidNumber;
buf[n] = c;
n += 1;
}
return self.intern(buf[0..n]);
}
/// Copy `s` into an AST-owned allocation.
fn intern(self: *Parser, s: []const u8) ParserError![]const u8 {
const owned = try self.allocator.dupe(u8, s);
errdefer self.allocator.free(owned);
try self.owned_strings.append(self.allocator, owned);
return owned;
}
// ── Datetime validation / classification ────────────────────────────────────
// The datetime subset of ExtKind; `classifyDatetime` only ever returns these
// four. (TOML never produces the enum/char-literal ExtKinds — those are ZON.)
const Shape = AST.Node.Kind.Extended.ExtKind;
/// Validate + classify a datetime token via the shared `util.datetime` helper.
/// TOML accepts a bare time and (in 1.1) minute precision; seconds are mandatory
/// in 1.0. The `util.datetime.Kind` lines up with `Shape` member-for-member.
fn classifyDatetime(self: *Parser, raw: []const u8) ParserError!Shape {
const kind = datetime.classify(raw, .{
.allow_minute_precision = self.version != .TOML_1_0,
}) catch return error.InvalidDatetime;
return switch (kind) {
.offset_datetime => .offset_datetime,
.local_datetime => .local_datetime,
.local_date => .local_date,
.local_time => .local_time,
};
}
// ── Tests ───────────────────────────────────────────────────────────────────
test "parses empty / comment-only documents" {
const inputs = [_][]const u8{ "", "# c\n", "\n\n \n" };
for (inputs) |input| {
var doc = try parse(testing.allocator, input, .TOML_1_0);
defer doc.deinit(testing.allocator);
try testing.expect(doc.ast.nodes[doc.ast.root].kind == .mapping);
}
}
test "parses scalar key/value pairs" {
var doc = try parse(testing.allocator,
\\name = "Tom"
\\count = 42
\\pi = 3.14
\\hex = 0xDEAD_beef
\\flag = true
\\when = 1979-05-27T07:32:00Z
\\
, .TOML_1_0);
defer doc.deinit(testing.allocator);
const ast = &doc.ast;
const name = try ast.getValByPath(&.{.{ .key = "name" }});
try testing.expectEqualStrings("Tom", name.kind.string);
const count = try ast.getValByPath(&.{.{ .key = "count" }});
try testing.expect(count.kind.number.kind == .integer);
const pi = try ast.getValByPath(&.{.{ .key = "pi" }});
try testing.expect(pi.kind.number.kind == .float);
const when = try ast.getValByPath(&.{.{ .key = "when" }});
try testing.expect(when.kind.extended.kind == .offset_datetime);
}
test "datetime shapes" {
const cases = [_]struct { src: []const u8, shape: Shape }{
.{ .src = "1979-05-27T07:32:00Z", .shape = .offset_datetime },
.{ .src = "1979-05-27T07:32:00", .shape = .local_datetime },
.{ .src = "1979-05-27 07:32:00", .shape = .local_datetime },
.{ .src = "1979-05-27", .shape = .local_date },
.{ .src = "07:32:00", .shape = .local_time },
.{ .src = "00:32:00.999999", .shape = .local_time },
};
for (cases) |c| {
const src = try std.fmt.allocPrint(testing.allocator, "d = {s}\n", .{c.src});
defer testing.allocator.free(src);
var doc = try parse(testing.allocator, src, .TOML_1_0);
defer doc.deinit(testing.allocator);
const d = try doc.ast.getValByPath(&.{.{ .key = "d" }});
try testing.expectEqual(c.shape, d.kind.extended.kind);
}
}
test "rejects bad scalars" {
const bad = [_][]const u8{
"x = 01\n", // leading zero
"x = 1__2\n", // doubled underscore
"x = 0x\n", // empty hex
"x = 1979-13-01\n", // month over
"x = 2021-02-29\n", // not a leap year
"x = 00:00:61\n", // second over
"x = \"a\nb\"\n", // newline in single-line string
"x = \"\\q\"\n", // bad escape
"a = 1 b = 2\n", // trailing content
};
for (bad) |input| {
if (parse(testing.allocator, input, .TOML_1_0)) |doc| {
var d = doc;
d.deinit(testing.allocator);
std.debug.print("expected rejection: {s}\n", .{input});
return error.ExpectedParseFailure;
} else |_| {}
}
}
test "rejects duplicate root keys" {
try testing.expectError(error.DuplicateKey, parse(testing.allocator, "a = 1\na = 2\n", .TOML_1_0));
}
test "tables and dotted keys build nested mappings" {
var doc = try parse(testing.allocator,
\\[server.tcp]
\\port = 80
\\opts.timeout = 30
\\
\\[server]
\\name = "main"
\\
, .TOML_1_0);
defer doc.deinit(testing.allocator);
const ast = &doc.ast;
const port = try ast.getValByPath(&.{ .{ .key = "server" }, .{ .key = "tcp" }, .{ .key = "port" } });
try testing.expect(port.kind.number.kind == .integer);
const timeout = try ast.getValByPath(&.{ .{ .key = "server" }, .{ .key = "tcp" }, .{ .key = "opts" }, .{ .key = "timeout" } });
try testing.expectEqualStrings("30", timeout.kind.number.raw);
const name = try ast.getValByPath(&.{ .{ .key = "server" }, .{ .key = "name" } });
try testing.expectEqualStrings("main", name.kind.string);
}
test "implicit table promoted to explicit is allowed" {
var doc = try parse(testing.allocator, "[a.b.c]\nx = 1\n\n[a]\ny = 2\n", .TOML_1_0);
defer doc.deinit(testing.allocator);
}
test "arrays, inline tables, and arrays-of-tables" {
var doc = try parse(testing.allocator,
\\nums = [1, 2, 3]
\\nested = [[1, 2], ["a", "b"]]
\\point = { x = 1, y = 2 }
\\
\\[[fruit]]
\\name = "apple"
\\
\\[[fruit]]
\\name = "pear"
\\
, .TOML_1_0);
defer doc.deinit(testing.allocator);
const ast = &doc.ast;
const n1 = try ast.getValByPath(&.{ .{ .key = "nums" }, .{ .index = 1 } });
try testing.expectEqualStrings("2", n1.kind.number.raw);
const inner = try ast.getValByPath(&.{ .{ .key = "nested" }, .{ .index = 1 }, .{ .index = 0 } });
try testing.expectEqualStrings("a", inner.kind.string);
const y = try ast.getValByPath(&.{ .{ .key = "point" }, .{ .key = "y" } });
try testing.expectEqualStrings("2", y.kind.number.raw);
const pear = try ast.getValByPath(&.{ .{ .key = "fruit" }, .{ .index = 1 }, .{ .key = "name" } });
try testing.expectEqualStrings("pear", pear.kind.string);
}
test "inline-table dotted keys" {
var doc = try parse(testing.allocator, "a = { b.c = 1, b.d = 2 }\n", .TOML_1_0);
defer doc.deinit(testing.allocator);
const c = try doc.ast.getValByPath(&.{ .{ .key = "a" }, .{ .key = "b" }, .{ .key = "c" } });
try testing.expectEqualStrings("1", c.kind.number.raw);
}
test "rejects inline-table and array errors" {
const bad = [_][]const u8{
"a = { b = 1, }\n", // trailing comma (1.0)
"a = { b = 1\n c = 2 }\n", // newline inside inline table (1.0)
"a = {b=1}\n[a.c]\nx=1\n", // extend a closed inline table
"a = [1, 2\n", // unclosed array
"a = { b = 1, b = 2 }\n", // duplicate inline key
};
for (bad) |input| {
if (parse(testing.allocator, input, .TOML_1_0)) |doc| {
var d = doc;
d.deinit(testing.allocator);
std.debug.print("expected rejection: {s}\n", .{input});
return error.ExpectedParseFailure;
} else |_| {}
}
}
test "TOML 1.1 features: optional seconds, \\e/\\x escapes, inline-table newlines" {
var doc = try parse(testing.allocator,
\\t = 13:37
\\esc = "\e\x41"
\\tbl = {
\\ a = 1,
\\ b = 2,
\\}
\\
, .TOML_1_1);
defer doc.deinit(testing.allocator);
const ast = &doc.ast;
const t = try ast.getValByPath(&.{.{ .key = "t" }});
try testing.expect(t.kind.extended.kind == .local_time);
const esc = try ast.getValByPath(&.{.{ .key = "esc" }});
try testing.expectEqualStrings("\x1bA", esc.kind.string);
const b = try ast.getValByPath(&.{ .{ .key = "tbl" }, .{ .key = "b" } });
try testing.expectEqualStrings("2", b.kind.number.raw);
}
test "1.1-only constructs are rejected under 1.0" {
const only_1_1 = [_][]const u8{
"t = 13:37\n", // optional seconds
"s = \"\\e\"\n", // \e escape
"s = \"\\x41\"\n", // \x escape
"t = { a = 1,\n b = 2 }\n", // newline in inline table
"t = { a = 1, }\n", // trailing comma
};
for (only_1_1) |input| {
// Valid under 1.1 …
var ok = try parse(testing.allocator, input, .TOML_1_1);
ok.deinit(testing.allocator);
// … but rejected under 1.0.
if (parse(testing.allocator, input, .TOML_1_0)) |doc| {
var d = doc;
d.deinit(testing.allocator);
std.debug.print("expected 1.0 rejection: {s}\n", .{input});
return error.ExpectedParseFailure;
} else |_| {}
}
}
test "rejects table/key conflicts" {
const bad = [_][]const u8{
"[a]\nb = 1\n\n[a.b]\nc = 2\n", // value used as table
"[a]\n\n[a]\n", // duplicate table
"a.b = 1\na.b.c = 2\n", // dotted key through a value
"[a.b.c]\nz = 9\n\n[a]\nb.c.t = 1\n", // dotted-append to explicit table
};
for (bad) |input| {
if (parse(testing.allocator, input, .TOML_1_0)) |doc| {
var d = doc;
d.deinit(testing.allocator);
std.debug.print("expected rejection: {s}\n", .{input});
return error.ExpectedParseFailure;
} else |_| {}
}
}
// ── diagnostics ──────────────────────────────────────────────────────────────
test "parseWithReport anchors the diagnostic on the offending token" {
var report: Report = .{};
// `a = ` — nothing follows the `=`; the newline is the unexpected token.
const src = "a = \nb = 1\n";
try testing.expectError(error.UnexpectedToken, Parser.parseWithReport(testing.allocator, src, .TOML_1_0, &report));
defer testing.allocator.free(report.warnings);
const d = report.diag.?;
try testing.expectEqual(error.UnexpectedToken, d.code);
try testing.expectEqual(@as(usize, 4), d.offset); // the newline right after `a = `
try testing.expectEqual(@as(usize, 0), report.errors.len); // single-shot: no recovery
}
test "parseWithReport pins a duplicate-key diagnostic on the key, not the value" {
var report: Report = .{};
const src = "a = 1\na = 2\n";
try testing.expectError(error.DuplicateKey, Parser.parseWithReport(testing.allocator, src, .TOML_1_0, &report));
defer testing.allocator.free(report.warnings);
const d = report.diag.?;
try testing.expectEqual(error.DuplicateKey, d.code);
try testing.expectEqual(@as(usize, 6), d.offset); // the second `a`, not `2`
}
test "parseWithReport reports the tokenizer's own failures with a location" {
var report: Report = .{};
const src = "a = \"unterminated\nb = 1\n";
try testing.expectError(error.UnclosedString, Parser.parseWithReport(testing.allocator, src, .TOML_1_0, &report));
defer testing.allocator.free(report.warnings);
const d = report.diag.?;
try testing.expectEqual(error.UnclosedString, d.code);
}
test "parseCollecting reports every broken statement in one pass" {
var report: Report = .{};
// Three independent broken lines interleaved with good ones.
const src = "a = \nb = 2\nc = \nd = 4\ne = \nf = 6\n";
defer if (report.errors.len > 0) testing.allocator.free(report.errors);
defer testing.allocator.free(report.warnings);
try testing.expectError(error.UnexpectedToken, Parser.parseCollecting(testing.allocator, src, .TOML_1_0, &report));
try testing.expectEqual(@as(usize, 3), report.errors.len);
var last_offset: usize = 0;
for (report.errors) |d| {
try testing.expectEqual(error.UnexpectedToken, d.code);
try testing.expect(d.offset >= last_offset);
last_offset = d.offset;
}
}
test "parseCollecting resyncs past a broken multi-line array without losing later statements" {
var report: Report = .{};
// The array never closes — `resync` must track bracket depth so the
// array's interior newlines don't look like safe statement boundaries,
// and must not loop forever when no depth-0 newline exists before EOF.
const src = "bad = [1, 2\ngood = 42\n";
defer if (report.errors.len > 0) testing.allocator.free(report.errors);
defer testing.allocator.free(report.warnings);
try testing.expectError(error.UnexpectedToken, Parser.parseCollecting(testing.allocator, src, .TOML_1_0, &report));
try testing.expect(report.errors.len >= 1);
}
test "an unclosed array doesn't cascade into false errors on every later well-formed statement" {
// The tokenizer runs once over the WHOLE file up front and is context-
// sensitive ([`/`{` nesting, key-vs-value mode); before `resync` also
// re-tokenized the recovered remainder, a genuinely unclosed `[` left
// that context stuck "inside a collection" for the rest of the file, so
// three unrelated, perfectly valid statements each spuriously errored too.
var report: Report = .{};
const src = "bad = [1, 2\nc = 3\nd = 4\ne = 5\n";
defer if (report.errors.len > 0) testing.allocator.free(report.errors);
defer testing.allocator.free(report.warnings);
try testing.expectError(error.UnexpectedToken, Parser.parseCollecting(testing.allocator, src, .TOML_1_0, &report));
// The unclosed array is one real defect; recovering past it should not
// multiply into one bogus error per subsequent line.
try testing.expectEqual(@as(usize, 1), report.errors.len);
}
test "parseCollecting still finds a second, genuinely separate error after an unclosed array" {
var report: Report = .{};
const src = "bad = [1, 2\ngood = 42\nbad2 = \nalsogood = 99\n";
defer if (report.errors.len > 0) testing.allocator.free(report.errors);
defer testing.allocator.free(report.warnings);
try testing.expectError(error.UnexpectedToken, Parser.parseCollecting(testing.allocator, src, .TOML_1_0, &report));
// One for the unclosed array (however it's attributed), one for `bad2 =`
// — but not a third for `alsogood`, which is perfectly valid.
try testing.expectEqual(@as(usize, 2), report.errors.len);
}
test "describe and shortLabel cover every Error variant" {
// Exhaustiveness is enforced by the compiler; this just checks a sample
// renders non-empty teaching text.
try testing.expect(describe(error.DuplicateKey).len > 0);
try testing.expect(shortLabel(error.InvalidNumber).len > 0);
}