//! YAML conformance scoreboard against the yaml-test-suite
//! (https://github.com/yaml/yaml-test-suite).
//!
//! Unlike the JSON harness, which hard-fails on any mismatch, this one is a
//! *scoreboard*: fig's YAML parser is still a subset of 1.2.2, so many accept
//! cases legitimately fail today. Each run prints a tally and asserts the score
//! has not dropped below a recorded baseline, so the numbers ratchet upward and
//! regressions are caught.
//!
//! Fixtures are generated by tools/gen_yaml_conformance.zig, which decodes the
//! suite's whitespace placeholders and excludes out-of-scope tests (anchors,
//! aliases, tags, directives) listed in testdata/yaml/skiplist.txt. Purely
//! multi-document streams are in scope via the Embed.extractStream splitter and
//! land in testdata/yaml/stream/, scored separately below.
//!
//! Run with: zig build test -Dyaml-conformance=true
const std = @import("std");
const testing = std.testing;
const AST = @import("../../ast/ast.zig");
const Parser = @import("parser.zig");
const Printer = @import("printer.zig");
const YamlType = @import("yaml.zig").Type;
const Embed = @import("../../embed.zig");
// The JSON side of the scoreboard (see `printsAsJson`). Imported by path, like
// `lossless.zig`'s test imports, so the count is scored even in a build that
// gates JSON out of the `Language` registry.
const JsonParser = @import("../json/parser.zig");
const JsonPrinter = @import("../json/printer.zig");
const materialize = @import("../../materialize.zig").materialize;
const max_fixture_size = 1024 * 1024;
// Baseline scores. These are a ratchet: raise them as coverage improves; never
// lower them without a deliberate reason. A run below baseline fails the test.
const accept_baseline = 289;
// Accept documents that also survive a print and re-parse: the printer must
// have a spelling for every node the parser can produce (a collection or
// alias as a mapping key, say), and that spelling must be YAML the parser
// reads back — including the `%TAG` directive that declares the handle a tag
// in the output is spelled with, without which the output is a document that
// uses an undeclared handle. Every accept fixture now clears it.
const reprint_baseline = 289;
// Accept documents that convert to JSON: materialized, printed as JSON, and the
// printed bytes read back by fig's own JSON parser — what `fig get -o json`
// does. Short of 289 by the 14 fixtures whose custom tag materialize refuses in
// strict mode — P76L among them, whose `%TAG !!` remaps the secondary handle so
// its `!!int` is custom — and the 15 whose mapping key is a collection (no
// JSON spelling, so `NonStringKey`). Raise this to 289 as those classes gain
// answers.
const json_baseline = 260;
const reject_baseline = 93;
// Multi-document streams parsed via Embed.extractStream (the single-document
// parser refuses a stream; the splitter feeds it one document at a time).
const stream_baseline = 19;
// Multi-document streams that must FAIL: Embed.extractStream must error on each
// (e.g. a tag handle defined only in the first document, used in a later one).
const reject_stream_baseline = 1;
const Score = struct {
correct: usize = 0,
total: usize = 0,
/// Of the correct `.should_pass` documents, how many printed and re-parsed.
reprinted: usize = 0,
/// Of the correct `.should_pass` documents, how many printed as JSON that
/// fig's JSON parser reads back (see `printsAsJson`).
json_printed: usize = 0,
};
test "yaml conformance: scoreboard" {
const accept = try scoreDir("testdata/yaml/accept", .should_pass);
const reject = try scoreDir("testdata/yaml/reject", .should_fail);
const stream = try scoreStreamDir("testdata/yaml/stream", .should_pass);
const reject_stream = try scoreStreamDir("testdata/yaml/reject-stream", .should_fail);
std.debug.print(
\\
\\YAML conformance (yaml-test-suite, out-of-scope excluded)
\\ accept (must parse): {d}/{d} baseline {d}
\\ reprint (print + re-parse): {d}/{d} baseline {d}
\\ json (materialize + print JSON + re-parse): {d}/{d} baseline {d}
\\ reject (must fail) : {d}/{d} baseline {d}
\\ stream (extractStream): {d}/{d} baseline {d}
\\ reject-stream (extractStream must fail): {d}/{d} baseline {d}
\\
, .{
accept.correct, accept.total, accept_baseline,
accept.reprinted, accept.total, reprint_baseline,
accept.json_printed, accept.total, json_baseline,
reject.correct, reject.total, reject_baseline,
stream.correct, stream.total, stream_baseline,
reject_stream.correct, reject_stream.total, reject_stream_baseline,
});
try testing.expect(accept.correct >= accept_baseline);
try testing.expect(accept.reprinted >= reprint_baseline);
try testing.expect(accept.json_printed >= json_baseline);
try testing.expect(reject.correct >= reject_baseline);
try testing.expect(stream.correct >= stream_baseline);
try testing.expect(reject_stream.correct >= reject_stream_baseline);
}
/// Print `ast` as YAML and parse the result. A printer without a spelling for
/// some node used to panic here (a non-string key); now it either spells it or
/// the re-parse fails and the document does not count.
fn reprints(ast: *const @import("../../ast/ast.zig")) !bool {
var out: std.Io.Writer.Allocating = .init(testing.allocator);
defer out.deinit();
try Printer.print(&out.writer, ast);
const again = Parser.parse(testing.allocator, out.written(), YamlType.v1_2_2) catch return false;
var d = again;
d.deinit(testing.allocator);
return true;
}
/// Materialize `ast` — aliases expanded, merges flattened, tags applied, which
/// is exactly what `fig get -o json` does to a YAML source — then print it as
/// JSON and parse those bytes back as JSON.
///
/// False when the document has no JSON reading at all: a custom tag strict
/// materialize refuses, or a collection used as a mapping key, which the JSON
/// printer answers with `NonStringKey` (a *scalar* key it spells as a string).
/// False, too — and this is what the ratchet is for — when JSON comes out that
/// fig's own JSON parser rejects, which is how a whole class of keys used to
/// leave here (`null: "a"`, `[ ... ]: 23`).
///
/// Every failure, allocation included, reads as "did not convert": a scoreboard
/// counts, and a test-only helper need not tell the reasons apart.
fn printsAsJson(ast: *const AST) bool {
var arena_state = std.heap.ArenaAllocator.init(testing.allocator);
defer arena_state.deinit();
const arena = arena_state.allocator();
const mat = materialize(arena, ast, .strict) catch return false;
var out: std.Io.Writer.Allocating = .init(arena);
JsonPrinter.print(&out.writer, &mat, .{}) catch return false;
// Arena-allocated: the parse result is freed with the arena, not by hand.
_ = JsonParser.parseAbstract(arena, out.written(), .JSON) catch return false;
return true;
}
/// Score the multi-document fixtures via Embed.extractStream, which errors if
/// any document fails. `.should_pass` fixtures must split + parse cleanly;
/// `.should_fail` fixtures must make the splitter error.
fn scoreStreamDir(dir_path: []const u8, expected: Expected) !Score {
var threaded = std.Io.Threaded.init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var dir = try std.Io.Dir.cwd().openDir(io, dir_path, .{ .iterate = true });
defer dir.close(io);
var score: Score = .{};
var iterator = dir.iterate();
while (try iterator.next(io)) |entry| {
if (entry.kind != .file) continue;
if (!std.mem.endsWith(u8, entry.name, ".yaml")) continue;
const input = try dir.readFileAlloc(io, entry.name, testing.allocator, .limited(max_fixture_size));
defer testing.allocator.free(input);
score.total += 1;
if (Embed.extractStream(testing.allocator, input)) |stream| {
stream.deinit(testing.allocator);
if (expected == .should_pass) score.correct += 1;
} else |_| {
if (expected == .should_fail) score.correct += 1;
}
}
return score;
}
const Expected = enum { should_pass, should_fail };
fn scoreDir(dir_path: []const u8, expected: Expected) !Score {
var threaded = std.Io.Threaded.init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var dir = try std.Io.Dir.cwd().openDir(io, dir_path, .{ .iterate = true });
defer dir.close(io);
var score: Score = .{};
var iterator = dir.iterate();
while (try iterator.next(io)) |entry| {
if (entry.kind != .file) continue;
if (!std.mem.endsWith(u8, entry.name, ".yaml")) continue;
const input = try dir.readFileAlloc(
io,
entry.name,
testing.allocator,
.limited(max_fixture_size),
);
defer testing.allocator.free(input);
score.total += 1;
const parsed = Parser.parse(testing.allocator, input, YamlType.v1_2_2);
switch (expected) {
.should_pass => {
if (parsed) |doc| {
var d = doc;
defer d.deinit(testing.allocator);
score.correct += 1;
if (try reprints(&d.ast)) score.reprinted += 1;
if (printsAsJson(&d.ast)) score.json_printed += 1;
} else |_| {}
},
.should_fail => {
if (parsed) |doc| {
var d = doc;
d.deinit(testing.allocator);
} else |_| {
score.correct += 1;
}
},
}
}
return score;
}