verit 0.2.0

//! `.verit` file performance — the curves a user would otherwise discover the
//! hard way.
//!
//!   cargo run --release -p verit --example filebench
//!
//! Three questions, because they are the three that decide how the format is
//! used and whether it needs to grow:
//!
//! 1. **Open latency vs record count.** `open` validates every index entry, so
//!    it is `O(records)` by design. How much does that cost?
//! 2. **Append cost vs commit batch size.** Each commit rewrites the whole
//!    index (spec §7.1), so committing per record is `O(n²)` overall and
//!    batching is `O(n)`. This is the curve that decides whether index-segment
//!    chaining is worth building.
//! 3. **Lookup.** `find_by_id` and `records_after` are binary searches over the
//!    index; confirm they behave like it.

use std::time::{Duration, Instant};

use verit::{Dt, FileBuilder, FileView, FileWriter, Schema, SchemaBuilder, Value};

fn schema() -> Schema {
    SchemaBuilder::new()
        .add_struct(
            "Event",
            vec![
                (1, "seq", Dt::U64),
                (2, "kind", Dt::Str),
                (3, "score", Dt::F64),
            ],
        )
        .build("Event")
        .unwrap()
}

fn event(i: u64) -> Value {
    Value::Struct(vec![
        (1, Value::U64(i)),
        (2, Value::str("checkpoint")),
        (3, Value::F64(i as f64 * 0.5)),
    ])
}

fn build(n: usize) -> Vec<u8> {
    let s = schema();
    let mut b = FileBuilder::new();
    for i in 0..n {
        b.append(&s, &event(i as u64)).unwrap();
    }
    b.finish().unwrap()
}

fn per(d: Duration, n: usize) -> String {
    if n == 0 {
        return "-".into();
    }
    let ns = d.as_secs_f64() * 1e9 / n as f64;
    if ns >= 1000.0 {
        format!("{:.1} µs", ns / 1000.0)
    } else {
        format!("{ns:.0} ns")
    }
}

fn main() {
    let dir = std::env::temp_dir().join(format!("verit-filebench-{}", std::process::id()));
    std::fs::create_dir_all(&dir).unwrap();

    // --- 1. open latency vs record count -----------------------------------
    println!("== open() vs record count ==");
    println!("  every index entry is validated on open, so this is O(records)\n");
    println!(
        "  {:>9}  {:>10}  {:>12}  {:>12}",
        "records", "file", "open", "per record"
    );
    for n in [100usize, 1_000, 10_000, 100_000] {
        let image = build(n);
        // Warm, then measure enough repeats to be stable.
        let reps = if n >= 10_000 { 20 } else { 200 };
        FileView::open(&image).unwrap();
        let t = Instant::now();
        for _ in 0..reps {
            let f = FileView::open(&image).unwrap();
            std::hint::black_box(f.len());
        }
        let d = t.elapsed() / reps;
        println!(
            "  {:>9}  {:>9}K  {:>12}  {:>12}",
            n,
            image.len() / 1024,
            per(d, 1),
            per(d, n)
        );
    }

    // --- 2. append cost vs commit batch size --------------------------------
    // Each commit rewrites the whole index, so N commits of 1 record is
    // O(N²) total index bytes and N/B commits of B records is O(N²/B).
    println!("\n== append: 10,000 records, varying records per commit ==");
    println!("  a commit rewrites the whole index (spec §7.1), so batching matters\n");
    println!(
        "  {:>10}  {:>9}  {:>12}  {:>14}",
        "per commit", "commits", "total", "per record"
    );
    const TOTAL: usize = 10_000;
    let s = schema();
    for batch in [1usize, 10, 100, 1_000, 10_000] {
        let path = dir.join(format!("append-{batch}.verit"));
        let _ = std::fs::remove_file(&path);
        let mut w = FileWriter::create(&path).unwrap();
        let t = Instant::now();
        let mut staged = 0;
        for i in 0..TOTAL {
            w.append(&s, &event(i as u64)).unwrap();
            staged += 1;
            if staged == batch {
                w.commit().unwrap();
                staged = 0;
            }
        }
        if staged > 0 {
            w.commit().unwrap();
        }
        let d = t.elapsed();
        println!(
            "  {:>10}  {:>9}  {:>12}  {:>14}",
            batch,
            TOTAL.div_ceil(batch),
            per(d, 1),
            per(d, TOTAL)
        );
        let _ = std::fs::remove_file(&path);
    }

    // --- 2b. is the index rewrite or the fsync the cost? --------------------
    // The roadmap assumed the O(records) index rewrite is what makes appends
    // expensive. Measure a fixed 1,000-record commit against files that already
    // hold 0 / 10k / 50k / 200k records: if the index rewrite dominates, the
    // cost rises with the file; if durability dominates, it stays flat.
    println!("\n== one 1,000-record commit, against a file that already holds N ==");
    println!("  rising cost => the index rewrite dominates (chaining would help)");
    println!("  flat cost   => fsync dominates (chaining would help nothing)\n");
    println!(
        "  {:>10}  {:>12}  {:>14}",
        "existing", "index bytes", "commit"
    );
    for existing in [0usize, 10_000, 50_000, 200_000] {
        let path = dir.join(format!("grow-{existing}.verit"));
        let _ = std::fs::remove_file(&path);
        let mut w = FileWriter::create(&path).unwrap();
        for i in 0..existing {
            w.append(&s, &event(i as u64)).unwrap();
        }
        if existing > 0 {
            w.commit().unwrap();
        }
        for i in 0..1000 {
            w.append(&s, &event(i as u64)).unwrap();
        }
        let t = Instant::now();
        w.commit().unwrap();
        let d = t.elapsed();
        println!(
            "  {:>10}  {:>11}K  {:>14}",
            existing,
            (existing + 1000) * 40 / 1024,
            per(d, 1)
        );
        let _ = std::fs::remove_file(&path);
    }

    // --- 3. lookup ----------------------------------------------------------
    println!("\n== lookup on a 100,000-record file ==");
    let image = build(100_000);
    let f = FileView::open(&image).unwrap();
    let ids: Vec<u64> = (0..1000).map(|k| (k * 97 % 100_000) as u64 + 1).collect();

    let t = Instant::now();
    for id in &ids {
        std::hint::black_box(f.find_by_id(*id));
    }
    println!(
        "  find_by_id (binary search)   {:>12}",
        per(t.elapsed(), ids.len())
    );

    let t = Instant::now();
    for _ in 0..1000 {
        std::hint::black_box(f.records_after(99_000).count());
    }
    println!(
        "  records_after, 1000 tail     {:>12}",
        per(t.elapsed(), 1000)
    );

    let t = Instant::now();
    for i in (0..100_000).step_by(97) {
        std::hint::black_box(f.get(i).unwrap().len());
    }
    println!(
        "  get(i) (zero-copy slice)     {:>12}",
        per(t.elapsed(), 100_000 / 97)
    );

    let _ = std::fs::remove_dir_all(&dir);
}