use std::time::Instant;
use jshift::{
for_each_jsonl_view_multi, for_each_page_items, JsonDoc, JsonlDecoder, JsonView, ObjectBuilder,
TypedDocRef,
};
use serde::Deserialize;
const WARMUP: usize = 3;
fn mean_ns(iters: usize, mut body: impl FnMut()) -> f64 {
for _ in 0..WARMUP {
body();
}
let t0 = Instant::now();
for _ in 0..iters {
body();
}
t0.elapsed().as_secs_f64() * 1e9 / iters as f64
}
fn fmt_time(ns: f64) -> String {
if ns < 1_000.0 {
format!("{ns:.0} ns")
} else if ns < 1_000_000.0 {
format!("{:.1} µs", ns / 1_000.0)
} else {
format!("{:.2} ms", ns / 1_000_000.0)
}
}
fn ratio(serde_ns: f64, j_ns: f64) -> String {
if j_ns <= 0.0 {
"∞".into()
} else {
format!("{:.1}×", serde_ns / j_ns)
}
}
fn row(name: &str, j: f64, s: f64) {
println!(
" {:<52} jshift {:>12} serde {:>12} serde/jshift {:>8}",
name,
fmt_time(j),
fmt_time(s),
ratio(s, j)
);
}
#[derive(Debug, JsonView)]
struct IdCard {
#[json(path = "id")]
id: u64,
}
#[derive(Debug, JsonView)]
struct ItemCard {
#[json(path = "id")]
id: u64,
#[json(path = "title")]
title: String,
}
#[derive(Debug, Deserialize)]
struct SerItem {
id: u64,
title: String,
#[allow(dead_code)]
noise: bool,
#[allow(dead_code)]
blob: String,
}
#[derive(Debug, Deserialize)]
struct SerPage {
items: Vec<SerItem>,
#[allow(dead_code)]
next: Option<u64>,
}
fn make_page(page: usize, items: usize, noise_blob: usize) -> Vec<u8> {
use jshift::ArrayBuilder;
let mut arr = ArrayBuilder::with_capacity(items * (64 + noise_blob));
for i in 0..items {
let id = (page * items + i) as u64;
arr = arr.object(|o| {
o.field("id", &id)
.field("title", &format!("item_{id}"))
.field("noise", &true)
.field("blob", &"x".repeat(noise_blob))
});
}
let items = arr.finish();
ObjectBuilder::with_capacity(items.len() + 32)
.raw_field("items", &items)
.field("next", &((page + 1) as u64))
.finish()
}
fn make_jsonl_shard(start_id: u64, n: usize, noise_blob: usize) -> Vec<u8> {
let mut out = Vec::with_capacity(n * (80 + noise_blob));
for i in 0..n {
let id = start_id + i as u64;
let line = ObjectBuilder::new()
.field("id", &id)
.field("title", &format!("item_{id}"))
.field("noise", &true)
.field("blob", &"x".repeat(noise_blob))
.finish();
out.extend_from_slice(&line);
out.push(b'\n');
}
out
}
fn main() {
println!("jshift multi-file / multi-page JSON vs serde_json\n");
let pages_n = 40usize;
let items_per = 250usize;
let blob = 256usize;
let pages: Vec<Vec<u8>> = (0..pages_n)
.map(|p| make_page(p, items_per, blob))
.collect();
let page_refs: Vec<&[u8]> = pages.iter().map(|p| p.as_slice()).collect();
let total_items = pages_n * items_per;
let pages_bytes: usize = pages.iter().map(|p| p.len()).sum();
println!(
"paginated: {pages_n} pages × {items_per} items (~{:.1} MiB total, {blob}B blob/item)\n",
pages_bytes as f64 / (1024.0 * 1024.0)
);
let j_sparse = mean_ns(25, || {
let mut sum = 0u64;
for_each_page_items::<IdCard, _, _, _>(page_refs.iter().copied(), "items", |_pos, row| {
sum = sum.wrapping_add(row.id);
Ok(())
})
.unwrap();
std::hint::black_box(sum);
});
let j_card = mean_ns(25, || {
let mut sum = 0u64;
for_each_page_items::<ItemCard, _, _, _>(page_refs.iter().copied(), "items", |_pos, row| {
sum = sum.wrapping_add(row.id);
sum = sum.wrapping_add(row.title.len() as u64);
Ok(())
})
.unwrap();
std::hint::black_box(sum);
});
let s = mean_ns(25, || {
let mut sum = 0u64;
for p in &pages {
let page: SerPage = serde_json::from_slice(p).unwrap();
for it in page.items {
sum = sum.wrapping_add(it.id);
sum = sum.wrapping_add(it.title.len() as u64);
}
}
std::hint::black_box(sum);
});
let j_each_get = mean_ns(25, || {
let mut sum = 0u64;
for p in page_refs.iter().copied() {
let doc = TypedDocRef::from_slice(p);
doc.each_get::<u64, _>("items", "id", |id| {
sum = sum.wrapping_add(id);
Ok(())
})
.unwrap();
}
std::hint::black_box(sum);
});
row(
&format!("pages: each_get id only ({total_items} rows)"),
j_each_get,
s,
);
row(
&format!("pages: IdCard view list ({total_items} rows)"),
j_sparse,
s,
);
row(
&format!("pages: ItemCard id+title ({total_items} rows)"),
j_card,
s,
);
let shards_n = 16usize;
let lines_per = 500usize;
let shards: Vec<Vec<u8>> = (0..shards_n)
.map(|s| make_jsonl_shard((s * lines_per) as u64, lines_per, blob))
.collect();
let shard_refs: Vec<&[u8]> = shards.iter().map(|s| s.as_slice()).collect();
let total_lines = shards_n * lines_per;
let shard_bytes: usize = shards.iter().map(|s| s.len()).sum();
println!(
"jsonl shards: {shards_n} files × {lines_per} lines (~{:.1} MiB total)\n",
shard_bytes as f64 / (1024.0 * 1024.0)
);
let j = mean_ns(20, || {
let mut sum = 0u64;
for_each_jsonl_view_multi::<IdCard, _, _, _>(shard_refs.iter().copied(), |_pos, row| {
sum = sum.wrapping_add(row.id);
Ok(())
})
.unwrap();
std::hint::black_box(sum);
});
let s = mean_ns(20, || {
let mut sum = 0u64;
for shard in &shards {
for line in shard.split(|&b| b == b'\n') {
if line.is_empty() {
continue;
}
let it: SerItem = serde_json::from_slice(line).unwrap();
sum = sum.wrapping_add(it.id);
}
}
std::hint::black_box(sum);
});
row(
&format!("multi JSONL sparse id ({total_lines} lines)"),
j,
s,
);
let stream = make_jsonl_shard(0, 3000, 128);
let chunk = 4096usize;
let j = mean_ns(25, || {
let mut dec = JsonlDecoder::with_capacity(chunk * 2);
let mut sum = 0u64;
let mut off = 0usize;
while off < stream.len() {
let end = (off + chunk).min(stream.len());
dec.push_and_for_each(&stream[off..end], |line| {
let row = IdCard::read_from(line).unwrap();
sum = sum.wrapping_add(row.id);
Ok(())
})
.unwrap();
off = end;
}
dec.finish(|line| {
let row = IdCard::read_from(line).unwrap();
sum = sum.wrapping_add(row.id);
Ok(())
})
.unwrap();
std::hint::black_box(sum);
});
let s = mean_ns(25, || {
let mut carry = Vec::new();
let mut sum = 0u64;
let mut off = 0usize;
while off < stream.len() {
let end = (off + chunk).min(stream.len());
carry.extend_from_slice(&stream[off..end]);
while let Some(nl) = carry.iter().position(|&b| b == b'\n') {
let line = &carry[..nl];
if !line.is_empty() {
let it: SerItem = serde_json::from_slice(line).unwrap();
sum = sum.wrapping_add(it.id);
}
carry.drain(..=nl);
}
off = end;
}
if !carry.is_empty() {
let it: SerItem = serde_json::from_slice(&carry).unwrap();
sum = sum.wrapping_add(it.id);
}
std::hint::black_box(sum);
});
row("chunked JSONL (4 KiB feeds, 3k lines, 128B blob)", j, s);
println!("\nNotes:");
println!(" • serde: full Deserialize of each page/line (allocates every field named on SerItem).");
println!(" • jshift: path view per row; multi-source never concatenates pages into one buffer.");
println!(" • JsonlDecoder: same carry-buffer idea as a careful serde loop; decode still sparse.");
println!(" • Fairness: both visit every record; jshift skips noise/blob tokens via path scan.");
}