jshift 0.7.0

Schema-guided typed JSON on raw bytes: path get/mutate, open views, project — without a Value tree.
Documentation
//! Multi-page JSON / multi-shard JSONL vs serde_json.
//!
//! ```sh
//! cargo run --release --example multi_json_bench --features derive
//! ```
//!
//! Scenarios (same data shape both stacks):
//! 1. **Paginated API** — N pages of `{"items":[{id, title, noise…}], "next":…}`
//! 2. **Sharded JSONL** — N buffers of M lines each
//! 3. **Chunked JSONL** — one logical stream split into small TCP-sized chunks
//!
//! serde approach: `from_slice` each page/line into typed structs (full token walk
//! per document). jshift: stream path views / line scans without a `Value` tree.

use std::time::Instant;

use jshift::{
    for_each_jsonl_view_multi, for_each_page_items, JsonDoc, JsonlDecoder, JsonView, ObjectBuilder,
    TypedDocRef,
};
use serde::Deserialize;

const WARMUP: usize = 3;

fn mean_ns(iters: usize, mut body: impl FnMut()) -> f64 {
    for _ in 0..WARMUP {
        body();
    }
    let t0 = Instant::now();
    for _ in 0..iters {
        body();
    }
    t0.elapsed().as_secs_f64() * 1e9 / iters as f64
}

fn fmt_time(ns: f64) -> String {
    if ns < 1_000.0 {
        format!("{ns:.0} ns")
    } else if ns < 1_000_000.0 {
        format!("{:.1} µs", ns / 1_000.0)
    } else {
        format!("{:.2} ms", ns / 1_000_000.0)
    }
}

fn ratio(serde_ns: f64, j_ns: f64) -> String {
    if j_ns <= 0.0 {
        "∞".into()
    } else {
        format!("{:.1}×", serde_ns / j_ns)
    }
}

fn row(name: &str, j: f64, s: f64) {
    println!(
        "  {:<52}  jshift {:>12}   serde {:>12}   serde/jshift {:>8}",
        name,
        fmt_time(j),
        fmt_time(s),
        ratio(s, j)
    );
}

/// Sparse jshift card (only paths we need).
#[derive(Debug, JsonView)]
struct IdCard {
    #[json(path = "id")]
    id: u64,
}

/// Fatter jshift card (still skips unknown keys not named here).
#[derive(Debug, JsonView)]
struct ItemCard {
    #[json(path = "id")]
    id: u64,
    #[json(path = "title")]
    title: String,
}

/// serde must name fields it wants; noise/blob still tokenized when present.
#[derive(Debug, Deserialize)]
struct SerItem {
    id: u64,
    title: String,
    #[allow(dead_code)]
    noise: bool,
    #[allow(dead_code)]
    blob: String,
}

#[derive(Debug, Deserialize)]
struct SerPage {
    items: Vec<SerItem>,
    #[allow(dead_code)]
    next: Option<u64>,
}

fn make_page(page: usize, items: usize, noise_blob: usize) -> Vec<u8> {
    use jshift::ArrayBuilder;
    let mut arr = ArrayBuilder::with_capacity(items * (64 + noise_blob));
    for i in 0..items {
        let id = (page * items + i) as u64;
        arr = arr.object(|o| {
            o.field("id", &id)
                .field("title", &format!("item_{id}"))
                .field("noise", &true)
                .field("blob", &"x".repeat(noise_blob))
        });
    }
    let items = arr.finish();
    ObjectBuilder::with_capacity(items.len() + 32)
        .raw_field("items", &items)
        .field("next", &((page + 1) as u64))
        .finish()
}

fn make_jsonl_shard(start_id: u64, n: usize, noise_blob: usize) -> Vec<u8> {
    let mut out = Vec::with_capacity(n * (80 + noise_blob));
    for i in 0..n {
        let id = start_id + i as u64;
        let line = ObjectBuilder::new()
            .field("id", &id)
            .field("title", &format!("item_{id}"))
            .field("noise", &true)
            .field("blob", &"x".repeat(noise_blob))
            .finish();
        out.extend_from_slice(&line);
        out.push(b'\n');
    }
    out
}

fn main() {
    println!("jshift multi-file / multi-page JSON vs serde_json\n");

    // ── Paginated API pages ─────────────────────────────────────────────────
    let pages_n = 40usize;
    let items_per = 250usize;
    let blob = 256usize;
    let pages: Vec<Vec<u8>> = (0..pages_n)
        .map(|p| make_page(p, items_per, blob))
        .collect();
    let page_refs: Vec<&[u8]> = pages.iter().map(|p| p.as_slice()).collect();
    let total_items = pages_n * items_per;
    let pages_bytes: usize = pages.iter().map(|p| p.len()).sum();
    println!(
        "paginated: {pages_n} pages × {items_per} items  (~{:.1} MiB total, {blob}B blob/item)\n",
        pages_bytes as f64 / (1024.0 * 1024.0)
    );

    let j_sparse = mean_ns(25, || {
        let mut sum = 0u64;
        for_each_page_items::<IdCard, _, _, _>(page_refs.iter().copied(), "items", |_pos, row| {
            sum = sum.wrapping_add(row.id);
            Ok(())
        })
        .unwrap();
        std::hint::black_box(sum);
    });
    let j_card = mean_ns(25, || {
        let mut sum = 0u64;
        for_each_page_items::<ItemCard, _, _, _>(page_refs.iter().copied(), "items", |_pos, row| {
            sum = sum.wrapping_add(row.id);
            sum = sum.wrapping_add(row.title.len() as u64);
            Ok(())
        })
        .unwrap();
        std::hint::black_box(sum);
    });
    let s = mean_ns(25, || {
        let mut sum = 0u64;
        for p in &pages {
            let page: SerPage = serde_json::from_slice(p).unwrap();
            for it in page.items {
                sum = sum.wrapping_add(it.id);
                sum = sum.wrapping_add(it.title.len() as u64);
            }
        }
        std::hint::black_box(sum);
    });
    let j_each_get = mean_ns(25, || {
        let mut sum = 0u64;
        for p in page_refs.iter().copied() {
            let doc = TypedDocRef::from_slice(p);
            doc.each_get::<u64, _>("items", "id", |id| {
                sum = sum.wrapping_add(id);
                Ok(())
            })
            .unwrap();
        }
        std::hint::black_box(sum);
    });
    row(
        &format!("pages: each_get id only ({total_items} rows)"),
        j_each_get,
        s,
    );
    row(
        &format!("pages: IdCard view list ({total_items} rows)"),
        j_sparse,
        s,
    );
    row(
        &format!("pages: ItemCard id+title ({total_items} rows)"),
        j_card,
        s,
    );

    // ── Multi JSONL shards ──────────────────────────────────────────────────
    let shards_n = 16usize;
    let lines_per = 500usize;
    let shards: Vec<Vec<u8>> = (0..shards_n)
        .map(|s| make_jsonl_shard((s * lines_per) as u64, lines_per, blob))
        .collect();
    let shard_refs: Vec<&[u8]> = shards.iter().map(|s| s.as_slice()).collect();
    let total_lines = shards_n * lines_per;
    let shard_bytes: usize = shards.iter().map(|s| s.len()).sum();
    println!(
        "jsonl shards: {shards_n} files × {lines_per} lines  (~{:.1} MiB total)\n",
        shard_bytes as f64 / (1024.0 * 1024.0)
    );

    let j = mean_ns(20, || {
        let mut sum = 0u64;
        for_each_jsonl_view_multi::<IdCard, _, _, _>(shard_refs.iter().copied(), |_pos, row| {
            sum = sum.wrapping_add(row.id);
            Ok(())
        })
        .unwrap();
        std::hint::black_box(sum);
    });
    let s = mean_ns(20, || {
        let mut sum = 0u64;
        for shard in &shards {
            for line in shard.split(|&b| b == b'\n') {
                if line.is_empty() {
                    continue;
                }
                let it: SerItem = serde_json::from_slice(line).unwrap();
                sum = sum.wrapping_add(it.id);
            }
        }
        std::hint::black_box(sum);
    });
    row(
        &format!("multi JSONL sparse id ({total_lines} lines)"),
        j,
        s,
    );

    // ── Chunked incremental JSONL (one big stream, 4 KiB chunks) ────────────
    let stream = make_jsonl_shard(0, 3000, 128);
    let chunk = 4096usize;
    let j = mean_ns(25, || {
        let mut dec = JsonlDecoder::with_capacity(chunk * 2);
        let mut sum = 0u64;
        let mut off = 0usize;
        while off < stream.len() {
            let end = (off + chunk).min(stream.len());
            dec.push_and_for_each(&stream[off..end], |line| {
                let row = IdCard::read_from(line).unwrap();
                sum = sum.wrapping_add(row.id);
                Ok(())
            })
            .unwrap();
            off = end;
        }
        dec.finish(|line| {
            let row = IdCard::read_from(line).unwrap();
            sum = sum.wrapping_add(row.id);
            Ok(())
        })
        .unwrap();
        std::hint::black_box(sum);
    });
    let s = mean_ns(25, || {
        // Careful serde: same carry-buffer framing, full Deserialize per line.
        let mut carry = Vec::new();
        let mut sum = 0u64;
        let mut off = 0usize;
        while off < stream.len() {
            let end = (off + chunk).min(stream.len());
            carry.extend_from_slice(&stream[off..end]);
            while let Some(nl) = carry.iter().position(|&b| b == b'\n') {
                let line = &carry[..nl];
                if !line.is_empty() {
                    let it: SerItem = serde_json::from_slice(line).unwrap();
                    sum = sum.wrapping_add(it.id);
                }
                carry.drain(..=nl);
            }
            off = end;
        }
        if !carry.is_empty() {
            let it: SerItem = serde_json::from_slice(&carry).unwrap();
            sum = sum.wrapping_add(it.id);
        }
        std::hint::black_box(sum);
    });
    row("chunked JSONL (4 KiB feeds, 3k lines, 128B blob)", j, s);

    println!("\nNotes:");
    println!("  • serde: full Deserialize of each page/line (allocates every field named on SerItem).");
    println!("  • jshift: path view per row; multi-source never concatenates pages into one buffer.");
    println!("  • JsonlDecoder: same carry-buffer idea as a careful serde loop; decode still sparse.");
    println!("  • Fairness: both visit every record; jshift skips noise/blob tokens via path scan.");
}