verit 0.2.0

//! `.verit` file conformance corpus (Rust side) — the fixtures every port's
//! reader is written against.
//!
//!   cargo run --example files -- emit   <dir>   # write the three corpora
//!   cargo run --example files -- verify <dir>   # check them
//!
//! Three corpora, because a `.verit` reader has three distinct obligations
//! (File Format Specification §11):
//!
//! - **`good/`** — valid files a reader must open and read correctly. Covers the
//!   empty file, one schema, mixed schemas, inline-schema records, a file at
//!   generation > 1, one with a removed record (dead bytes still present), and a
//!   compacted one (ids preserved, generation reset).
//! - **`torn/`** — files truncated mid-commit. Each must open at the *previous*
//!   generation and read correctly. This is the crash-safety guarantee as files
//!   on disk, so a port proves it rather than assuming it.
//! - **`bad/`** — one file per rejection rule, named for the rule. Each must be
//!   refused with a typed error, never a panic or a partial read.
//!
//! `FileBuilder` is deterministic, so `emit` run twice produces identical bytes
//! and the committed corpus is reviewable as a diff.
//!
//! Regenerate intentionally with:
//!   cargo run -p verit --example files -- emit engine/tests/files
//! and review the diff before committing.

use std::fs;
use std::path::{Path, PathBuf};

use verit::{
    encode, Dt, FileBuilder, FileView, FileWriter, Schema, SchemaBuilder, SchemaMode, Value,
};

// Footer and index-entry geometry, per spec §6 and §7. Spelled out here rather
// than imported so the corpus generator breaks loudly if the layout ever moves.
const FOOTER_LEN: usize = 64;
const INDEX_ENTRY: usize = 40;

// ---------------------------------------------------------------------------
// schemas
// ---------------------------------------------------------------------------

fn point_schema() -> Schema {
    SchemaBuilder::new()
        .add_struct("Point", vec![(1, "x", Dt::I32), (2, "y", Dt::I32)])
        .build("Point")
        .unwrap()
}

fn person_schema() -> Schema {
    SchemaBuilder::new()
        .add_struct(
            "Person",
            vec![
                (1, "name", Dt::Str),
                (2, "age", Dt::U8),
                (3, "tags", Dt::list(Dt::Str)),
            ],
        )
        .build("Person")
        .unwrap()
}

/// A schema carrying custom scalar defaults. The defaults live in the canonical
/// schema, so a `.verit` file's schema section carries them too — and a reader
/// that bounds its defaults decode with the *record's* envelope length instead
/// of the *section entry's* will silently skip them. Nothing else in the corpus
/// catches that.
fn defaults_schema() -> Schema {
    SchemaBuilder::new()
        .add_struct(
            "Settings",
            vec![
                (1, "name", Dt::Str),
                (2, "retries", Dt::U8),
                (3, "ratio", Dt::F64),
            ],
        )
        .set_default("Settings", 2, Value::U8(3))
        .set_default("Settings", 3, Value::F64(0.75))
        .build("Settings")
        .unwrap()
}

/// A narrower, older `Person` — the evolution case, so a port can prove an old
/// record still resolves out of a file that also holds the new schema.
fn person_v1_schema() -> Schema {
    SchemaBuilder::new()
        .add_struct("Person", vec![(1, "name", Dt::Str), (2, "age", Dt::U8)])
        .build("Person")
        .unwrap()
}

fn point(x: i32, y: i32) -> Value {
    Value::Struct(vec![(1, Value::I32(x)), (2, Value::I32(y))])
}

fn person(name: &str, age: u8, tags: &[&str]) -> Value {
    Value::Struct(vec![
        (1, Value::str(name)),
        (2, Value::U8(age)),
        (3, Value::List(tags.iter().map(|t| Value::str(t)).collect())),
    ])
}

// ---------------------------------------------------------------------------
// the good corpus
// ---------------------------------------------------------------------------

/// Files built purely in memory — deterministic, generation 1.
fn built_cases() -> Vec<(&'static str, Vec<u8>)> {
    let points = point_schema();
    let people = person_schema();
    let people_v1 = person_v1_schema();
    let mut out = Vec::new();

    out.push(("empty", FileBuilder::new().finish().unwrap()));

    let mut b = FileBuilder::new();
    b.append(&points, &point(3, 4)).unwrap();
    out.push(("single", b.finish().unwrap()));

    // Varying record lengths, to force padding between 8-aligned records.
    let mut b = FileBuilder::new();
    for n in 0..16u8 {
        b.append(&people, &person(&"x".repeat(n as usize), n, &["a"]))
            .unwrap();
    }
    out.push(("many", b.finish().unwrap()));

    // Three schemas interleaved, including two versions of the same type — the
    // evolution-at-rest case.
    let mut b = FileBuilder::new();
    b.append(&points, &point(1, 1)).unwrap();
    b.append(&people, &person("Ada", 36, &["math"])).unwrap();
    b.append(
        &people_v1,
        &Value::Struct(vec![(1, Value::str("Grace")), (2, Value::U8(45))]),
    )
    .unwrap();
    b.append(&points, &point(-2, 7)).unwrap();
    out.push(("mixed", b.finish().unwrap()));

    // Records carrying their own inline schema: legal, and a reader must handle
    // them even though writers emit hash-only.
    let mut b = FileBuilder::new();
    b.append_self_describing(&encode(&points, &point(9, 9), SchemaMode::Inline).unwrap())
        .unwrap();
    b.append_self_describing(
        &encode(&people, &person("Inline", 1, &[]), SchemaMode::Inline).unwrap(),
    )
    .unwrap();
    out.push(("inline", b.finish().unwrap()));

    // Custom scalar defaults: the record omits both defaulted fields, so a
    // reader must recover them from the schema section to render the record.
    let settings = defaults_schema();
    let mut b = FileBuilder::new();
    b.append(&settings, &Value::Struct(vec![(1, Value::str("alpha"))]))
        .unwrap();
    b.append(
        &settings,
        &Value::Struct(vec![(1, Value::str("beta")), (2, Value::U8(9))]),
    )
    .unwrap();
    out.push(("defaults", b.finish().unwrap()));

    out
}

/// Files that can only be produced by the on-disk writer, because they exercise
/// multiple generations, removal, and compaction.
fn writer_cases(scratch: &Path) -> Vec<(&'static str, Vec<u8>)> {
    let points = point_schema();
    let mut out = Vec::new();

    // Generation 4: three separate commits, so a reader must find the newest
    // footer past two superseded ones.
    let path = scratch.join("appended.build");
    let mut w = FileWriter::create(&path).unwrap();
    for i in 0..3 {
        w.append(&points, &point(i, i * 10)).unwrap();
        w.commit().unwrap();
    }
    out.push(("appended", fs::read(&path).unwrap()));

    // A removed record: ids have a gap, and the removed bytes are still in the
    // file. A reader must report the live set, not what it can find in the bytes.
    let path = scratch.join("removed.build");
    let mut w = FileWriter::create(&path).unwrap();
    let ids: Vec<u64> = (0..5)
        .map(|i| w.append(&points, &point(i, -i)).unwrap())
        .collect();
    w.commit().unwrap();
    w.remove_ids(&[ids[1], ids[3]]);
    w.commit().unwrap();
    out.push(("removed", fs::read(&path).unwrap()));

    // Compacted: generation back to 1, ids preserved with their gaps, and
    // next_record_id still ahead of every id ever issued.
    let path = scratch.join("compacted.build");
    let mut w = FileWriter::create(&path).unwrap();
    let ids: Vec<u64> = (0..5)
        .map(|i| w.append(&points, &point(i * 2, i)).unwrap())
        .collect();
    w.commit().unwrap();
    w.remove_ids(&[ids[0], ids[4]]);
    w.commit().unwrap();
    w.compact().unwrap();
    out.push(("compacted", fs::read(&path).unwrap()));

    out
}

// ---------------------------------------------------------------------------
// the torn corpus
// ---------------------------------------------------------------------------

/// A committed generation-2 file, then a large generation-3 commit truncated at
/// points inside each of its regions. Every one must roll back to generation 2.
fn torn_cases(scratch: &Path) -> Vec<(String, Vec<u8>, u64, usize)> {
    let points = point_schema();
    let path = scratch.join("torn.build");
    let mut w = FileWriter::create(&path).unwrap();
    w.append(&points, &point(1, 1)).unwrap();
    w.append(&points, &point(2, 2)).unwrap();
    w.commit().unwrap();
    let safe_len = fs::read(&path).unwrap().len();

    for n in 0..8 {
        w.append(&points, &point(100 + n, -n)).unwrap();
    }
    w.commit().unwrap();
    let full = fs::read(&path).unwrap();

    let footer_at = full.len() - FOOTER_LEN;
    let view = FileView::open(&full).unwrap();
    let index_at = view.footer().index_offset as usize;
    let schema_at = view.footer().schema_offset as usize;

    // One cut inside each region of the interrupted commit, plus the nastiest
    // case: everything written but the last byte of the footer.
    let cuts: Vec<(&str, usize)> = vec![
        ("torn-in-records", safe_len + 16),
        ("torn-at-schema-start", schema_at),
        ("torn-in-schema", schema_at + 8),
        ("torn-at-index-start", index_at),
        ("torn-in-index", index_at + INDEX_ENTRY + 8),
        ("torn-at-footer-start", footer_at),
        ("torn-in-footer", footer_at + 32),
        ("torn-one-byte-short", full.len() - 1),
    ];

    cuts.into_iter()
        .filter(|(_, at)| *at > safe_len && *at < full.len())
        .map(|(name, at)| (name.to_string(), full[..at].to_vec(), 2u64, 2usize))
        .collect()
}

// ---------------------------------------------------------------------------
// the bad corpus
// ---------------------------------------------------------------------------

/// Reseal a mutated footer so the file gets past the commit-integrity check and
/// is caught by the *structural* rules — which is the path that matters.
fn reseal(image: &mut [u8]) {
    let f = image.len() - FOOTER_LEN;
    let crc = verit::hash::crc32(&image[f..f + 56]);
    image[f + 56..f + 60].copy_from_slice(&crc.to_le_bytes());
}

fn bad_cases() -> Vec<(&'static str, &'static str, Vec<u8>)> {
    let points = point_schema();
    let mut b = FileBuilder::new();
    b.append(&points, &point(1, 2)).unwrap();
    b.append(&points, &point(3, 4)).unwrap();
    let good = b.finish().unwrap();

    let view = FileView::open(&good).unwrap();
    let entry = view.footer().index_offset as usize;
    let footer = good.len() - FOOTER_LEN;
    let mut out: Vec<(&'static str, &'static str, Vec<u8>)> = Vec::new();

    let mut m = |name, why, f: &dyn Fn(&mut Vec<u8>)| {
        let mut image = good.clone();
        f(&mut image);
        out.push((name, why, image));
    };

    m("bad-magic", "bytes 0..4 are not VRTF", &|i| i[0] = b'X');
    m("bad-version", "unsupported file version", &|i| i[4] = 2);
    m(
        "nonzero-reserved-header",
        "reserved header byte is not zero",
        &|i| i[5] = 1,
    );
    m(
        "unknown-required-feature",
        "required_features bit this build does not implement",
        &|i| i[8..12].copy_from_slice(&4u32.to_le_bytes()),
    );
    m(
        "misaligned-index-entry",
        "record offset is not 8-aligned",
        &|i| i[entry + 8..entry + 16].copy_from_slice(&33u64.to_le_bytes()),
    );
    m(
        "entry-overlaps-header",
        "record offset is inside the 32-byte header",
        &|i| i[entry + 8..entry + 16].copy_from_slice(&0u64.to_le_bytes()),
    );
    m(
        "entry-past-records",
        "record extends beyond the record region",
        &|i| i[entry + 8..entry + 16].copy_from_slice(&u64::MAX.to_le_bytes()),
    );
    m("entry-huge-length", "record length overflows", &|i| {
        i[entry + 16..entry + 24].copy_from_slice(&u64::MAX.to_le_bytes())
    });
    m(
        "missing-schema",
        "index references a schema the section does not carry",
        &|i| i[entry + 24] ^= 0x01,
    );
    m(
        "forged-record-count",
        "record_count does not fit between index and footer",
        &|i| {
            i[footer + 28..footer + 32].copy_from_slice(&u32::MAX.to_le_bytes());
            reseal(i);
        },
    );
    m(
        "index-not-at-footer",
        "index does not end exactly at the footer",
        &|i| {
            i[footer + 8..footer + 16].copy_from_slice(&(entry as u64 + 8).to_le_bytes());
            reseal(i);
        },
    );
    m("zero-record-id", "record id 0 is reserved", &|i| {
        i[entry..entry + 8].copy_from_slice(&0u64.to_le_bytes());
        reseal(i);
    });
    m(
        "descending-record-ids",
        "ids are not strictly ascending",
        &|i| {
            i[entry..entry + 8].copy_from_slice(&9u64.to_le_bytes());
            reseal(i);
        },
    );
    m("duplicate-record-id", "two records share one id", &|i| {
        i[entry + INDEX_ENTRY..entry + INDEX_ENTRY + 8].copy_from_slice(&1u64.to_le_bytes());
        reseal(i);
    });
    m(
        "stale-next-record-id",
        "next_record_id does not exceed every live id",
        &|i| {
            i[footer + 40..footer + 48].copy_from_slice(&1u64.to_le_bytes());
            reseal(i);
        },
    );
    m(
        "bad-footer-crc",
        "footer CRC does not match (a torn commit with no fallback)",
        &|i| i[footer + 56] ^= 0xFF,
    );
    m(
        "no-valid-footer",
        "trailing magic destroyed and nothing else to fall back to",
        &|i| i[footer + 60..footer + 64].copy_from_slice(b"XXXX"),
    );

    // Not a mutation of a good file: a 0.1.0 container, which must be refused
    // structurally rather than partially misread.
    #[allow(deprecated)]
    {
        let mut c = verit::ContainerWriter::new();
        c.add(b"\x00not-a-verit-file");
        out.push((
            "vertc-container",
            "a 0.1.0 .vertc container, not a .verit file",
            c.finish(),
        ));
    }
    out.push((
        "truncated-header",
        "shorter than the 32-byte header",
        good[..16].to_vec(),
    ));
    out.push(("empty-file", "zero bytes", Vec::new()));

    out
}

// ---------------------------------------------------------------------------
// emit / verify
// ---------------------------------------------------------------------------

fn emit(dir: &Path) {
    let scratch = std::env::temp_dir().join(format!("verit-files-emit-{}", std::process::id()));
    let _ = fs::remove_dir_all(&scratch);
    fs::create_dir_all(&scratch).unwrap();

    for sub in ["good", "torn", "bad"] {
        let d = dir.join(sub);
        let _ = fs::remove_dir_all(&d);
        fs::create_dir_all(&d).unwrap();
    }

    // --- good ---
    let good_dir = dir.join("good");
    let mut manifest = String::from("# file\trecords\tgeneration\tnext_record_id\n");
    let mut cases = built_cases();
    cases.extend(writer_cases(&scratch));
    for (name, image) in &cases {
        fs::write(good_dir.join(format!("{name}.verit")), image).unwrap();
        let f = FileView::open(image).unwrap();
        manifest.push_str(&format!(
            "{name}\t{}\t{}\t{}\n",
            f.len(),
            f.generation(),
            f.next_record_id()
        ));
        // One row per record: id, schema id, and the JSON the file alone yields.
        let mut records = String::from("# record_id\tschema_id\tdump_json\n");
        for i in 0..f.len() {
            records.push_str(&format!(
                "{}\t{:032x}\t{}\n",
                f.record(i).unwrap().id,
                f.schema_id(i).unwrap(),
                f.dump_json(i).unwrap()
            ));
        }
        fs::write(good_dir.join(format!("{name}.records.tsv")), records).unwrap();
    }
    fs::write(good_dir.join("manifest.tsv"), manifest).unwrap();

    // --- torn ---
    let torn_dir = dir.join("torn");
    let mut manifest = String::from("# file\texpected_generation\texpected_records\n");
    let torn = torn_cases(&scratch);
    let torn_count = torn.len();
    for (name, image, gen, records) in torn {
        fs::write(torn_dir.join(format!("{name}.verit")), &image).unwrap();
        manifest.push_str(&format!("{name}\t{gen}\t{records}\n"));
    }
    fs::write(torn_dir.join("manifest.tsv"), manifest).unwrap();

    // --- bad ---
    let bad_dir = dir.join("bad");
    let mut manifest = String::from("# file\trule violated\n");
    let bad = bad_cases();
    let bad_count = bad.len();
    for (name, why, image) in bad {
        fs::write(bad_dir.join(format!("{name}.verit")), &image).unwrap();
        manifest.push_str(&format!("{name}\t{why}\n"));
    }
    fs::write(bad_dir.join("manifest.tsv"), manifest).unwrap();

    let _ = fs::remove_dir_all(&scratch);
    println!(
        "files: emitted {} good, {torn_count} torn, {bad_count} bad to {}",
        cases.len(),
        dir.display()
    );
}

fn rows(path: &Path) -> Vec<Vec<String>> {
    fs::read_to_string(path)
        .unwrap_or_else(|e| panic!("{}: {e}", path.display()))
        .lines()
        .filter(|l| !l.starts_with('#') && !l.trim().is_empty())
        .map(|l| l.split('\t').map(|c| c.to_string()).collect())
        .collect()
}

fn verify(dir: &Path) -> bool {
    let mut passed = 0usize;
    let mut failed = 0usize;
    let mut check = |ok: bool, what: String| {
        if ok {
            passed += 1;
            println!("files verify: {what} PASS");
        } else {
            failed += 1;
            println!("files verify: {what} FAIL");
        }
    };

    // --- good: open, and match every file-level and per-record fact ---
    let good = dir.join("good");
    for row in rows(&good.join("manifest.tsv")) {
        let (name, records, generation, next_id) = (
            &row[0],
            row[1].parse::<usize>().unwrap(),
            row[2].parse::<u64>().unwrap(),
            row[3].parse::<u64>().unwrap(),
        );
        let image = fs::read(good.join(format!("{name}.verit"))).unwrap();
        let f = match FileView::open(&image) {
            Ok(f) => f,
            Err(e) => {
                check(false, format!("good/{name} (open failed: {e})"));
                continue;
            }
        };
        let mut ok =
            f.len() == records && f.generation() == generation && f.next_record_id() == next_id;
        for (i, r) in rows(&good.join(format!("{name}.records.tsv")))
            .iter()
            .enumerate()
        {
            let id = r[0].parse::<u64>().unwrap();
            ok &= f.record(i).map(|rec| rec.id) == Ok(id);
            // Found by id, not by position — the identity contract.
            ok &= f.find_by_id(id) == Some(i);
            ok &= format!("{:032x}", f.schema_id(i).unwrap()) == r[1];
            ok &= f.dump_json(i).unwrap() == r[2];
        }
        check(ok, format!("good/{name}"));
    }

    // --- torn: must recover the previous generation and still read ---
    let torn = dir.join("torn");
    for row in rows(&torn.join("manifest.tsv")) {
        let (name, generation, records) = (
            &row[0],
            row[1].parse::<u64>().unwrap(),
            row[2].parse::<usize>().unwrap(),
        );
        let image = fs::read(torn.join(format!("{name}.verit"))).unwrap();
        let ok = match FileView::open(&image) {
            Ok(f) => {
                let structural = f.generation() == generation && f.len() == records;
                // Not merely openable: the rolled-back records must still read.
                structural && (0..f.len()).all(|i| f.dump_json(i).is_ok())
            }
            Err(_) => false,
        };
        check(ok, format!("torn/{name}"));
    }

    // --- bad: must be refused with a typed error, never a panic ---
    let bad = dir.join("bad");
    for row in rows(&bad.join("manifest.tsv")) {
        let name = &row[0];
        let image = fs::read(bad.join(format!("{name}.verit"))).unwrap();
        let ok = matches!(
            std::panic::catch_unwind(|| FileView::open(&image).map(|f| f.len())),
            Ok(Err(_))
        );
        check(ok, format!("bad/{name}"));
    }

    println!("files verify: {passed} passed, {failed} failed");
    failed == 0
}

fn main() {
    let args: Vec<String> = std::env::args().collect();
    let usage = "usage: files <emit|verify> <dir>";
    let (mode, dir) = match (args.get(1).map(String::as_str), args.get(2)) {
        (Some(m), Some(d)) => (m, PathBuf::from(d)),
        _ => {
            eprintln!("{usage}");
            std::process::exit(2);
        }
    };
    match mode {
        "emit" => {
            fs::create_dir_all(&dir).unwrap();
            emit(&dir);
        }
        "verify" => {
            if !verify(&dir) {
                std::process::exit(1);
            }
        }
        _ => {
            eprintln!("{usage}");
            std::process::exit(2);
        }
    }
}