fig 1.0.0

//! YAML conformance scoreboard against the yaml-test-suite
//! (https://github.com/yaml/yaml-test-suite).
//!
//! Unlike the JSON harness, which hard-fails on any mismatch, this one is a
//! *scoreboard*: fig's YAML parser is still a subset of 1.2.2, so many accept
//! cases legitimately fail today. Each run prints a tally and asserts the score
//! has not dropped below a recorded baseline, so the numbers ratchet upward and
//! regressions are caught.
//!
//! Fixtures are generated by tools/gen_yaml_conformance.zig, which decodes the
//! suite's whitespace placeholders and excludes out-of-scope tests (anchors,
//! aliases, tags, directives) listed in testdata/yaml/skiplist.txt. Purely
//! multi-document streams are in scope via the Embed.extractStream splitter and
//! land in testdata/yaml/stream/, scored separately below.
//!
//! Run with: zig build test -Dyaml-conformance=true

const std = @import("std");
const testing = std.testing;

const Parser = @import("parser.zig");
const YamlType = @import("yaml.zig").Type;
const Embed = @import("../embed.zig");

const max_fixture_size = 1024 * 1024;

// Baseline scores. These are a ratchet: raise them as coverage improves; never
// lower them without a deliberate reason. A run below baseline fails the test.
const accept_baseline = 289;
const reject_baseline = 93;
// Multi-document streams parsed via Embed.extractStream (the single-document
// parser refuses a stream; the splitter feeds it one document at a time).
const stream_baseline = 19;
// Multi-document streams that must FAIL: Embed.extractStream must error on each
// (e.g. a tag handle defined only in the first document, used in a later one).
const reject_stream_baseline = 1;

const Score = struct {
    correct: usize = 0,
    total: usize = 0,
};

test "yaml conformance: scoreboard" {
    const accept = try scoreDir("testdata/yaml/accept", .should_pass);
    const reject = try scoreDir("testdata/yaml/reject", .should_fail);
    const stream = try scoreStreamDir("testdata/yaml/stream", .should_pass);
    const reject_stream = try scoreStreamDir("testdata/yaml/reject-stream", .should_fail);

    std.debug.print(
        \\
        \\YAML conformance (yaml-test-suite, out-of-scope excluded)
        \\  accept (must parse): {d}/{d}   baseline {d}
        \\  reject (must fail) : {d}/{d}   baseline {d}
        \\  stream (extractStream): {d}/{d}   baseline {d}
        \\  reject-stream (extractStream must fail): {d}/{d}   baseline {d}
        \\
    , .{
        accept.correct,        accept.total,        accept_baseline,
        reject.correct,        reject.total,        reject_baseline,
        stream.correct,        stream.total,        stream_baseline,
        reject_stream.correct, reject_stream.total, reject_stream_baseline,
    });

    try testing.expect(accept.correct >= accept_baseline);
    try testing.expect(reject.correct >= reject_baseline);
    try testing.expect(stream.correct >= stream_baseline);
    try testing.expect(reject_stream.correct >= reject_stream_baseline);
}

/// Score the multi-document fixtures via Embed.extractStream, which errors if
/// any document fails. `.should_pass` fixtures must split + parse cleanly;
/// `.should_fail` fixtures must make the splitter error.
fn scoreStreamDir(dir_path: []const u8, expected: Expected) !Score {
    var threaded = std.Io.Threaded.init(testing.allocator, .{});
    defer threaded.deinit();
    const io = threaded.io();

    var dir = try std.Io.Dir.cwd().openDir(io, dir_path, .{ .iterate = true });
    defer dir.close(io);

    var score: Score = .{};
    var iterator = dir.iterate();
    while (try iterator.next(io)) |entry| {
        if (entry.kind != .file) continue;
        if (!std.mem.endsWith(u8, entry.name, ".yaml")) continue;

        const input = try dir.readFileAlloc(io, entry.name, testing.allocator, .limited(max_fixture_size));
        defer testing.allocator.free(input);

        score.total += 1;
        if (Embed.extractStream(testing.allocator, input)) |stream| {
            stream.deinit(testing.allocator);
            if (expected == .should_pass) score.correct += 1;
        } else |_| {
            if (expected == .should_fail) score.correct += 1;
        }
    }
    return score;
}

const Expected = enum { should_pass, should_fail };

fn scoreDir(dir_path: []const u8, expected: Expected) !Score {
    var threaded = std.Io.Threaded.init(testing.allocator, .{});
    defer threaded.deinit();
    const io = threaded.io();

    var dir = try std.Io.Dir.cwd().openDir(io, dir_path, .{ .iterate = true });
    defer dir.close(io);

    var score: Score = .{};
    var iterator = dir.iterate();
    while (try iterator.next(io)) |entry| {
        if (entry.kind != .file) continue;
        if (!std.mem.endsWith(u8, entry.name, ".yaml")) continue;

        const input = try dir.readFileAlloc(
            io,
            entry.name,
            testing.allocator,
            .limited(max_fixture_size),
        );
        defer testing.allocator.free(input);

        score.total += 1;
        const parsed = Parser.parse(testing.allocator, input, YamlType.v1_2_2);
        switch (expected) {
            .should_pass => {
                if (parsed) |doc| {
                    var d = doc;
                    d.deinit(testing.allocator);
                    score.correct += 1;
                } else |_| {}
            },
            .should_fail => {
                if (parsed) |doc| {
                    var d = doc;
                    d.deinit(testing.allocator);
                } else |_| {
                    score.correct += 1;
                }
            },
        }
    }
    return score;
}